diff --git a/.github/harness/prompts/review.md b/.github/harness/prompts/review.md index 0a4f85fc7..d34c67b95 100644 --- a/.github/harness/prompts/review.md +++ b/.github/harness/prompts/review.md @@ -3,11 +3,16 @@ Review this GitHub PR: {pr_url} You have tools to fetch the PR diff, read files, search the web, and post comments on the PR. You have these repos cloned locally for context: + - /opt/workspace/agentcore-cli — aws/agentcore-cli - /opt/workspace/agentcore-l3-cdk-constructs — aws/agentcore-l3-cdk-constructs -Before reviewing, read all existing comments on the PR to understand what has already been discussed. Do not repeat or re-post issues that have already been raised in existing comments. +Before reviewing, read all existing comments on the PR to understand what has already been discussed. Do not repeat or +re-post issues that have already been raised in existing comments. -Review the PR. If there are any serious issues that require code changes before merging, post a comment on the PR for each issue explaining the problem. If there are multiple ways to fix an issue, list the options so the author can choose. Skip style nits and minor suggestions — only flag things that actually need to change. +Review the PR. If there are any serious issues that require code changes before merging, post a comment on the PR for +each issue explaining the problem. If there are multiple ways to fix an issue, list the options so the author can +choose. Skip style nits and minor suggestions — only flag things that actually need to change. -If all serious issues have already been raised in existing comments, or if you found no new issues, post a single comment on the PR saying it looks good to merge (or that all issues have already been flagged). +If all serious issues have already been raised in existing comments, or if you found no new issues, post a single +comment on the PR saying it looks good to merge (or that all issues have already been flagged). diff --git a/.github/harness/prompts/system.md b/.github/harness/prompts/system.md index 963accb8a..52a3d2260 100644 --- a/.github/harness/prompts/system.md +++ b/.github/harness/prompts/system.md @@ -6,11 +6,13 @@ This workspace contains two repos for developing and testing the AgentCore CLI. ### agentcore-cli/ (`aws/agentcore-cli`) -The terminal experience for creating, developing, and deploying AI agents to AgentCore. Node.js/TypeScript CLI built with Ink (React-based TUI). +The terminal experience for creating, developing, and deploying AI agents to AgentCore. Node.js/TypeScript CLI built +with Ink (React-based TUI). ### agentcore-l3-cdk-constructs/ (`aws/agentcore-l3-cdk-constructs`) -AWS CDK L3 constructs for declaring and deploying AgentCore infrastructure. Used by agentcore-cli to vend CDK projects when users run `agentcore create`. +AWS CDK L3 constructs for declaring and deploying AgentCore infrastructure. Used by agentcore-cli to vend CDK projects +when users run `agentcore create`. ## How they relate @@ -18,4 +20,6 @@ AWS CDK L3 constructs for declaring and deploying AgentCore infrastructure. Used ## Testing with a bundled distribution -Run `npm run bundle` in `agentcore-cli/` to create a tar distribution that includes the packaged `agentcore-l3-cdk-constructs`. You can then install it globally with `npm install -g ` to test the CLI end-to-end. +Run `npm run bundle` in `agentcore-cli/` to create a tar distribution that includes the packaged +`agentcore-l3-cdk-constructs`. You can then install it globally with `npm install -g ` to test the CLI +end-to-end. diff --git a/.github/scripts/prompts/review.md b/.github/scripts/prompts/review.md new file mode 100644 index 000000000..0a4f85fc7 --- /dev/null +++ b/.github/scripts/prompts/review.md @@ -0,0 +1,13 @@ +Review this GitHub PR: {pr_url} + +You have tools to fetch the PR diff, read files, search the web, and post comments on the PR. + +You have these repos cloned locally for context: +- /opt/workspace/agentcore-cli — aws/agentcore-cli +- /opt/workspace/agentcore-l3-cdk-constructs — aws/agentcore-l3-cdk-constructs + +Before reviewing, read all existing comments on the PR to understand what has already been discussed. Do not repeat or re-post issues that have already been raised in existing comments. + +Review the PR. If there are any serious issues that require code changes before merging, post a comment on the PR for each issue explaining the problem. If there are multiple ways to fix an issue, list the options so the author can choose. Skip style nits and minor suggestions — only flag things that actually need to change. + +If all serious issues have already been raised in existing comments, or if you found no new issues, post a single comment on the PR saying it looks good to merge (or that all issues have already been flagged). diff --git a/.github/scripts/prompts/system.md b/.github/scripts/prompts/system.md new file mode 100644 index 000000000..963accb8a --- /dev/null +++ b/.github/scripts/prompts/system.md @@ -0,0 +1,21 @@ +# AgentCore CLI Development Workspace + +This workspace contains two repos for developing and testing the AgentCore CLI. + +## Repositories + +### agentcore-cli/ (`aws/agentcore-cli`) + +The terminal experience for creating, developing, and deploying AI agents to AgentCore. Node.js/TypeScript CLI built with Ink (React-based TUI). + +### agentcore-l3-cdk-constructs/ (`aws/agentcore-l3-cdk-constructs`) + +AWS CDK L3 constructs for declaring and deploying AgentCore infrastructure. Used by agentcore-cli to vend CDK projects when users run `agentcore create`. + +## How they relate + +`agentcore-cli` is the main product. It vends CDK projects using constructs from `agentcore-l3-cdk-constructs`. + +## Testing with a bundled distribution + +Run `npm run bundle` in `agentcore-cli/` to create a tar distribution that includes the packaged `agentcore-l3-cdk-constructs`. You can then install it globally with `npm install -g ` to test the CLI end-to-end. diff --git a/.github/scripts/python/harness_review.py b/.github/scripts/python/harness_review.py new file mode 100644 index 000000000..fbfd0b0f9 --- /dev/null +++ b/.github/scripts/python/harness_review.py @@ -0,0 +1,217 @@ +"""Invoke Bedrock AgentCore Harness to review a GitHub PR. + +Reads PR_URL from the environment. Streams harness output to stdout. +Uses raw HTTP with SigV4 signing — no custom service model needed. +""" + +import json +import os +import sys +import time +import uuid + +import boto3 +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.eventstream import EventStreamBuffer +from urllib.parse import quote +import urllib3 + +# ANSI color codes +CYAN = "\033[36m" +YELLOW = "\033[33m" +GREEN = "\033[32m" +RED = "\033[31m" +DIM = "\033[2m" +RESET = "\033[0m" + +SCRIPTS_DIR = os.path.join(os.path.dirname(__file__), "..") + + +def read_prompt(filename): + """Read a prompt template from the prompts directory.""" + path = os.path.join(SCRIPTS_DIR, "prompts", filename) + with open(path) as f: + return f.read() + + +def invoke_harness(harness_arn, body, region): + """Send a SigV4-signed request to the harness invoke endpoint. Returns a streaming response. + + InvokeHarness is not in standard boto3, so we call the REST API directly. + boto3 is only used to resolve AWS credentials (from env vars, OIDC, etc.) + and sign the request with SigV4. The response is an AWS binary event stream. + """ + session = boto3.Session(region_name=region) + credentials = session.get_credentials().get_frozen_credentials() + url = f"https://bedrock-agentcore.{region}.amazonaws.com/harnesses/invoke?harnessArn={quote(harness_arn, safe='')}" + request = AWSRequest(method="POST", url=url, data=body, headers={ + "Content-Type": "application/json", + "Accept": "application/vnd.amazon.eventstream", + }) + SigV4Auth(credentials, "bedrock-agentcore", region).add_auth(request) + return urllib3.PoolManager().urlopen( + "POST", url, body=body, + headers=dict(request.headers), + preload_content=False, + timeout=urllib3.Timeout(connect=10, read=600), + ) + + +def parse_events(http_response): + """Yield (event_type, payload) tuples from the harness binary event stream. + + The response arrives as raw bytes in AWS binary event stream format. + EventStreamBuffer reassembles complete events from the 4KB chunks, + and we decode each event's JSON payload before yielding it. + """ + event_buffer = EventStreamBuffer() + for chunk in http_response.stream(4096): + event_buffer.add_data(chunk) + for event in event_buffer: + if event.headers.get(":message-type") == "exception": + payload = json.loads(event.payload.decode("utf-8")) + print(f"\n{RED}ERROR: {payload}{RESET}", file=sys.stderr) + sys.exit(1) + event_type = event.headers.get(":event-type", "") + if event.payload: + yield event_type, json.loads(event.payload.decode("utf-8")) + + +def print_stream(http_response): + """Display harness events with GitHub Actions log groups. + + The harness streams events as the agent works: + contentBlockStart — a new block begins (text or tool call) + contentBlockDelta — incremental chunks of text or tool input JSON + contentBlockStop — block complete, we now have full tool input to display + messageStop — agent finished + internalServerException — server error + + Tool calls are wrapped in ::group::/::endgroup:: for collapsible sections + in the GitHub Actions log UI. Agent reasoning text is printed inline in dim. + """ + start_time = time.time() + iteration = 0 + tool_name = None + tool_input = "" + tool_start = 0.0 + in_group = False + text_buffer = "" + + def close_group(): + nonlocal in_group + if in_group: + print("::endgroup::", flush=True) + in_group = False + + def flush_text(): + nonlocal text_buffer + if text_buffer: + for line in text_buffer.splitlines(): + print(f"{DIM}{line}{RESET}", flush=True) + text_buffer = "" + + for event_type, payload in parse_events(http_response): + + if event_type == "contentBlockStart": + start = payload.get("start", {}) + if "toolUse" in start: + tool_name = start["toolUse"].get("name", "unknown") + tool_input = "" + tool_start = time.time() + iteration += 1 + + elif event_type == "contentBlockDelta": + delta = payload.get("delta", {}) + if "text" in delta: + close_group() + text_buffer += delta["text"] + if "toolUse" in delta: + tool_input += delta["toolUse"].get("input", "") + + elif event_type == "contentBlockStop": + flush_text() + if tool_name: + elapsed = time.time() - tool_start + try: + parsed = json.loads(tool_input) + except (json.JSONDecodeError, TypeError): + parsed = tool_input + + close_group() + + cmd = parsed.get("command") if isinstance(parsed, dict) else None + header = f"{CYAN}[{iteration}]{RESET} {YELLOW}{tool_name}{RESET} {DIM}({elapsed:.1f}s){RESET}" + if cmd: + header += f": $ {cmd}" + + print(f"::group::{header}", flush=True) + in_group = True + + if isinstance(parsed, dict): + for k, v in parsed.items(): + if k != "command": + print(f" {DIM}{k}:{RESET} {str(v)[:300]}", flush=True) + + tool_name = None + tool_input = "" + + elif event_type == "messageStop": + flush_text() + close_group() + if payload.get("stopReason") == "end_turn": + total = time.time() - start_time + print(f"\n\n{GREEN}{'=' * 50}", flush=True) + print(f" Done ({int(total // 60)}m {int(total % 60)}s)", flush=True) + print(f"{'=' * 50}{RESET}", flush=True) + + elif event_type == "internalServerException": + close_group() + print(f"\n{RED}ERROR: {payload}{RESET}", file=sys.stderr) + sys.exit(1) + + close_group() + total = time.time() - start_time + print(f"\n{GREEN}Review complete.{RESET} {DIM}({iteration} tool calls, {int(total)}s total){RESET}") + + +# --- Main --- + +# All config comes from environment variables (set via GitHub secrets/workflow) +MODEL_ID = os.environ.get("HARNESS_MODEL_ID", "us.anthropic.claude-opus-4-7") +HARNESS_ARN = os.environ.get("HARNESS_ARN", "") +PR_URL = os.environ.get("PR_URL", "") + +for name, val in [("HARNESS_ARN", HARNESS_ARN), ("PR_URL", PR_URL)]: + if not val: + print(f"{RED}ERROR: {name} environment variable is required{RESET}", file=sys.stderr) + sys.exit(1) + +# Extract region from the ARN (arn:aws:bedrock-agentcore:{region}:{account}:harness/{id}) +REGION = HARNESS_ARN.split(":")[3] +SESSION_ID = str(uuid.uuid4()).upper() + +print(f"{CYAN}Session:{RESET} {SESSION_ID}") +print(f"{CYAN}PR:{RESET} {PR_URL}") +print(f"{CYAN}Harness:{RESET} {HARNESS_ARN}") +print() + +SYSTEM_PROMPT = read_prompt("system.md") +REVIEW_PROMPT = read_prompt("review.md").format(pr_url=PR_URL) + +request_body = json.dumps({ + "runtimeSessionId": SESSION_ID, + "systemPrompt": [{"text": SYSTEM_PROMPT}], + "messages": [{"role": "user", "content": [{"text": REVIEW_PROMPT}]}], + "model": {"bedrockModelConfig": {"modelId": MODEL_ID}}, +}) + +http_response = invoke_harness(HARNESS_ARN, request_body, REGION) + +if http_response.status != 200: + error = http_response.read().decode("utf-8") + print(f"{RED}ERROR: HTTP {http_response.status}: {error}{RESET}", file=sys.stderr) + sys.exit(1) + +print_stream(http_response) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 80b17f987..e6d7bf8cc 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -2,9 +2,9 @@ name: Build and Test on: push: - branches: ['main'] + branches: ['main', 'preview'] pull_request: - branches: ['main'] + branches: ['main', 'preview'] permissions: contents: read @@ -12,7 +12,7 @@ permissions: # Cancel in-progress runs for PRs; never cancel runs on main (merges should not abort each other) concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/preview' }} jobs: build: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1e9b0a4bd..2a01ca7fa 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -2,16 +2,16 @@ name: CodeQL on: push: - branches: ['main'] + branches: ['main', 'preview'] pull_request: - branches: ['main'] + branches: ['main', 'preview'] pull_request_target: - branches: ['main'] + branches: ['main', 'preview'] # Cancel in-progress runs for PRs; never cancel runs on main (merges should not abort each other) concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/preview' }} jobs: analyze: @@ -19,6 +19,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 permissions: + actions: read security-events: write contents: read diff --git a/.github/workflows/e2e-tests-full.yml b/.github/workflows/e2e-tests-full.yml index 14809a587..35f92ad59 100644 --- a/.github/workflows/e2e-tests-full.yml +++ b/.github/workflows/e2e-tests-full.yml @@ -8,7 +8,7 @@ on: schedule: - cron: '0 14 * * 1' # Every Monday at 9 AM EST (14:00 UTC) push: - branches: [main] + branches: [main, preview] concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} @@ -27,10 +27,11 @@ jobs: fail-fast: false matrix: cdk-source: [npm, main] + shard: ['1/6', '2/6', '3/6', '4/6', '5/6', '6/6'] steps: - uses: actions/checkout@v6 with: - ref: ${{ github.event_name == 'workflow_dispatch' && github.ref || 'main' }} + ref: ${{ github.sha }} - uses: actions/setup-node@v6 with: node-version: '20.x' @@ -49,7 +50,7 @@ jobs: id: aws run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT" - name: Get API keys from Secrets Manager - uses: aws-actions/aws-secretsmanager-get-secrets@v3 + uses: aws-actions/aws-secretsmanager-get-secrets@v2 with: secret-ids: | E2E,${{ secrets.E2E_SECRET_ARN }} @@ -70,7 +71,7 @@ jobs: CDK_REPO: ${{ secrets.CDK_REPO_NAME }} - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}) + - name: Run E2E tests (${{ matrix.cdk-source }}, shard ${{ matrix.shard }}) env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -78,7 +79,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - run: npm run test:e2e + run: npx vitest run --project e2e --shard=${{ matrix.shard }} browser-tests: runs-on: ubuntu-latest environment: e2e-testing @@ -86,7 +87,7 @@ jobs: steps: - uses: actions/checkout@v6 with: - ref: ${{ github.event_name == 'workflow_dispatch' && github.ref || 'main' }} + ref: ${{ github.sha }} - uses: actions/setup-node@v6 with: node-version: '20.x' diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index e0ee36b5b..78f38eefd 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -6,7 +6,7 @@ on: description: 'AWS region for deployment' default: 'us-east-1' pull_request_target: - branches: [main] + branches: [main, preview] concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} @@ -70,7 +70,7 @@ jobs: id: aws run: echo "account_id=$(aws sts get-caller-identity --query Account --output text)" >> "$GITHUB_OUTPUT" - name: Get API keys from Secrets Manager - uses: aws-actions/aws-secretsmanager-get-secrets@v3 + uses: aws-actions/aws-secretsmanager-get-secrets@v2 with: secret-ids: | E2E,${{ secrets.E2E_SECRET_ARN }} @@ -101,6 +101,7 @@ jobs: BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }} CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | grep -v '^e2e-tests/harness-bedrock\.test\.ts$' \ | tr '\n' ' ') echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT" echo "Changed e2e tests: ${CHANGED:-none}" @@ -113,5 +114,7 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR - run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }} + # Always run strands-bedrock and harness-bedrock as baseline, plus any e2e test files changed in the PR + run: + npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts e2e-tests/harness-bedrock.test.ts ${{ + steps.changed.outputs.extra_tests }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 24a9317f6..f310a776a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,9 +2,9 @@ name: Quality and Safety Checks on: push: - branches: ['main'] + branches: ['main', 'preview'] pull_request: - branches: ['main'] + branches: ['main', 'preview'] permissions: contents: read @@ -12,7 +12,7 @@ permissions: # Cancel in-progress runs for PRs; never cancel runs on main (merges should not abort each other) concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/preview' }} jobs: setup: @@ -109,7 +109,7 @@ jobs: fetch-depth: 0 - name: Reject schema changes outside release PRs run: | - if git diff --name-only origin/main...HEAD | grep -q '^schemas/agentcore\.schema\.v[0-9]*\.json$'; then + if git diff --name-only origin/${{ github.base_ref || github.ref_name }}...HEAD | grep -q '^schemas/agentcore\.schema\.v[0-9]*\.json$'; then echo "" echo "❌ schemas/ must not be modified directly." echo "The JSON schema is served live from the repo — changes are released automatically." diff --git a/.github/workflows/pr-size.yml b/.github/workflows/pr-size.yml index 792f0d728..520a44fe0 100644 --- a/.github/workflows/pr-size.yml +++ b/.github/workflows/pr-size.yml @@ -4,7 +4,7 @@ name: PR Size Check and Label # Safe because this workflow only reads PR metadata — it never checks out untrusted code. on: pull_request_target: - branches: [main] + branches: [main, preview] jobs: label-size: diff --git a/.github/workflows/pr-tarball.yml b/.github/workflows/pr-tarball.yml index 3c5c5c522..a901a2a6e 100644 --- a/.github/workflows/pr-tarball.yml +++ b/.github/workflows/pr-tarball.yml @@ -1,7 +1,7 @@ name: PR Tarball on: pull_request_target: - branches: [main] + branches: [main, preview] permissions: contents: write diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 4b0953754..d06a5e609 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -2,7 +2,7 @@ name: Validate PR Title on: pull_request_target: - branches: [main] + branches: [main, preview] types: [opened, edited, synchronize, reopened] permissions: diff --git a/.github/workflows/slack-issue-notification.yml b/.github/workflows/slack-issue-notification.yml index 758add1d1..1d3bbc4ee 100644 --- a/.github/workflows/slack-issue-notification.yml +++ b/.github/workflows/slack-issue-notification.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Send issue details to Slack - uses: slackapi/slack-github-action@v3.0.2 + uses: slackapi/slack-github-action@v2.1.1 with: webhook: ${{ secrets.SLACK_WEBHOOK_URL }} webhook-type: webhook-trigger diff --git a/.github/workflows/slack-open-prs-notification.yml b/.github/workflows/slack-open-prs-notification.yml index 11641bed4..68dd1df49 100644 --- a/.github/workflows/slack-open-prs-notification.yml +++ b/.github/workflows/slack-open-prs-notification.yml @@ -40,7 +40,7 @@ jobs: ); - name: Send open PRs summary to Slack - uses: slackapi/slack-github-action@v3.0.2 + uses: slackapi/slack-github-action@v2.1.1 with: webhook: ${{ secrets.SLACK_OPEN_PRS_WEBHOOK_URL }} webhook-type: webhook-trigger diff --git a/.github/workflows/sync-from-public.yml b/.github/workflows/sync-from-public.yml new file mode 100644 index 000000000..94e279079 --- /dev/null +++ b/.github/workflows/sync-from-public.yml @@ -0,0 +1,104 @@ +name: Sync from Public Repo + +on: + schedule: + - cron: '0 */6 * * *' # Every 6 hours + workflow_dispatch: # Manual trigger via Actions tab + +permissions: + contents: write + pull-requests: write + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Fetch public main + run: | + git remote add public https://github.com/aws/agentcore-cli.git + git fetch public main + + - name: Sync main with public/main + run: | + git checkout main + git reset --hard origin/main + + # Check if public/main is already merged + if git merge-base --is-ancestor public/main HEAD; then + echo "✅ main is already up to date with public/main" + exit 0 + fi + + # Merge but exclude .github/workflows/ (GITHUB_TOKEN lacks workflow permission) + if git merge public/main --no-commit --no-ff; then + git checkout HEAD -- .github/workflows/ 2>/dev/null || true + git commit -m "chore: sync main with public/main" + git push origin main + echo "✅ main synced successfully" + else + echo "⚠️ Conflict detected in main" + + # Capture conflicted files before aborting + conflicted_files=$(git diff --name-only --diff-filter=U 2>/dev/null || echo "Unable to determine conflicted files") + git merge --abort + + # Check if a sync PR already exists + existing_pr=$(gh pr list --base "main" --search "Merge public/main" --state open --json number --jq '.[0].number' 2>/dev/null || echo "") + + if [ -n "$existing_pr" ]; then + echo "ℹ️ PR #$existing_pr already exists, skipping" + exit 0 + fi + + conflict_branch="sync-conflict-main-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$conflict_branch" + + git merge public/main --no-commit --no-ff || true + git checkout HEAD -- .github/workflows/ 2>/dev/null || true + git add -A + git commit -m "chore: sync main with public/main (conflicts present) + + This automated sync detected merge conflicts that require manual resolution. + + Source: public/main (https://github.com/aws/agentcore-cli) + Target: main + + Please resolve conflicts and merge this PR." || true + + git push origin "$conflict_branch" + + gh pr create \ + --title "🔀 [Sync Conflict] Merge public/main → main" \ + --body "## Automated Sync Conflict + + This PR was automatically created because merging \`public/main\` into \`main\` encountered conflicts. + + **Source:** \`main\` from [aws/agentcore-cli](https://github.com/aws/agentcore-cli) + **Target:** \`main\` + + ### Action Required + 1. \`git fetch origin && git checkout $conflict_branch\` + 2. Resolve merge conflicts + 3. \`git add . && git commit\` + 4. \`git push origin $conflict_branch\` + 5. Merge this PR + + ### Files with Conflicts + \`\`\` + $conflicted_files + \`\`\`" \ + --base "main" \ + --head "$conflict_branch" || echo "⚠️ Failed to create PR" + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index ac3d53fc4..6613a8f02 100644 --- a/.gitignore +++ b/.gitignore @@ -69,11 +69,6 @@ ProtocolTesting/ .cdk-constructs-clone/ .omc/ -# E2E test artifacts -e2e-tests/fixtures/import/bugbash-resources.json - -# oh-my-claudecode -.omc/ # Browser tests browser-tests/.browser-test-env browser-tests/test-results/ diff --git a/.prettierignore b/.prettierignore index 3b1452b18..8eda17e39 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,3 @@ CHANGELOG.md src/assets/**/*.md -.github/harness/prompts/ +.github/scripts/prompts/ diff --git a/CHANGELOG.md b/CHANGELOG.md index b25ef328d..b99c235b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,18 @@ All notable changes to this project will be documented in this file. -## [0.12.2] - 2026-04-30 +## [1.0.0-preview.6] - 2026-05-01 + +### Added +- feat: replace credentialProviderName with outboundAuth for harness gateway tools (#1083) (f818286) + +### Fixed +- fix: restore preview-specific package.json, lock, and changelog (#1084) (b873a49) + +### Other Changes +- chore: sync main into preview — evo features (#1077) (aeb796a) + +## [1.0.0-preview.5] - 2026-04-30 ### Added - feat: add telemetry audit mode with FileSystemSink (#1014) (397c187) @@ -11,13 +22,20 @@ All notable changes to this project will be documented in this file. - fix: add Accept header to HTTP protocol invocation proxy (#1051) (821e4c3) ### Other Changes +- Merge pull request #1057 from aws/sync-preview/merge-main-20260430-v2 (18fa2c9) +- chore: merge main into preview (7590650) +- Merge pull request #1058 from aws/release/v0.12.2 (68b25bf) +- chore: bump version to 0.12.2 (5ce4bdc) +- chore: merge main into preview (6e01e4e) - fix(harness): add error handling for invoke_harness API call (#1056) (9a6a5d0) - Merge pull request #1054 from aws/fix/remove-coauthor-reland (0afeaf5) - refactor: move harness resources to .github/harness/ and use boto3 invoke_harness (ad2ba9b) - Revert "refactor: move harness resources to .github/harness/ (#992)" (b8a90c9) - refactor: move harness resources to .github/harness/ (#992) (aef3890) +- Merge pull request #1053 from aws/sync-preview/merge-main-20260430 (26b1c4c) +- chore: merge main into preview (9f2702a) -## [0.12.1] - 2026-04-29 +## [1.0.0-preview.4] - 2026-04-29 ### Added - feat: add CloudWatch traces API for web UI (#997) (76b07aa) @@ -26,9 +44,17 @@ All notable changes to this project will be documented in this file. - fix: remove CONFIG_DIR exclusion from zip stage to preserve dependency agentcore/ packages (#1015) (d1e5241) ### Other Changes +- Merge pull request #1040 from aws/sync-preview/merge-main-20260429-v5 (dd76d17) +- chore: merge main into preview (ecda10c) +- fix(ci): install uv in release workflow prepare steps (#1038) (#1039) (01b3b7d) - fix(ci): install uv in release workflow prepare steps (#1038) (29ae8e5) +- Merge pull request #1037 from aws/sync-preview/merge-main-20260429-v3 (7f315c6) +- chore: merge main into preview (a951aed) - fix(ci): move snapshot update after build in release workflow (#1036) (227c840) +- Merge pull request #1035 from aws/sync-preview/merge-main-20260429-v2 (451868a) +- chore: merge main into preview (c44d8c1) - fix(ci): enable coverage collection in sharded unit test runs (#1034) (061b6b3) +- ci: run all PR and merge workflows on preview branch (#1023) (fc1cd56) - fix(ci): update snapshots after CDK version sync in release workflow (#1033) (d3b412f) - chore(deps): bump @opentelemetry/sdk-metrics from 2.6.1 to 2.7.0 (#1030) (ad59fc0) - chore(deps-dev): bump secretlint from 12.2.0 to 12.3.1 (#1029) (36755e9) @@ -36,22 +62,40 @@ All notable changes to this project will be documented in this file. - chore(deps): bump @opentelemetry/resources from 2.6.1 to 2.7.0 (#1026) (ad482cf) - chore(deps): bump the aws-cdk group with 2 updates (#1025) (1686e4d) - chore(deps): bump the aws-sdk group with 14 updates (#1024) (1fc366c) +- Merge pull request #1018 from aws/sync-preview/real-merge-main-20260429 (8c4d6eb) +- chore: merge main into preview (553a520) +- sync-preview: merge main into preview (#1017) (1c726d8) - ci: add coordinated main + preview release workflow (#995) (7e8cae4) +- chore: merge main into preview (#1013) (3e7e15b) - fix(import): use GatewayNameSchema for gateway import name validation (#1011) (29b6522) - test: remove 44 render-only and framework-testing tests (#998) (13b34a3) +- chore: bump version to 0.12.0 (#1002) (dd9270d) -## [0.12.0] - 2026-04-28 +## [1.0.0-preview.3] - 2026-04-28 ### Added - feat: add gateway import command with executionRoleArn support (#855) (2df1387) - feat: runtime endpoint support in AgentCore CLI (#979) (41c59ef) - feat: add project-name option to create (#969) (9b46fbb) +- feat: add project-name option to preview create (#970) (a19fc8f) +- feat: add agentcore-cli User-Agent to all API calls (#960) (398dc50) +- feat: add telemetry schemas and client (#941) (7c37fa6) +- feat: add GitHub Action for automated PR review via AgentCore Harness (#934) (a365bf5) ### Fixed - fix: duplicate header flash and help menu truncation (closes #895, closes #637) (#955) (e7b85c1) - fix: show 'Computing diff changes...' step during deploy diff phase (#952) (a725d12) +- fix: display session ID after CLI invoke completes (#957) (51e4a8e) +- fix: lower eventExpiryDuration minimum from 7 to 3 days (closes #744) (#956) (8613657) +- fix: use pull_request_target for fork PR support (#958) (933bac8) +- fix: agentcore dev not working in windows (#951) (5271f55) +- fix: add TTY detection before TUI fallbacks to prevent agent/CI hangs (#949) (c30ed54) +- fix: allow code-based evaluators in online eval configs (#947) (3d2d671) +- fix: buffer streaming text to avoid per-token log lines in GitHub Actions (#946) (cb1e81a) ### Other Changes +- fix(tests): fix 2 test failures on preview branch (8a4ea58) +- Merge main into preview (3fd6668) - fix(e2e): add debug logging for gateway import CI failures (#1001) (8012d6c) - fix(e2e): separate gateway import test and add PR-changed test detection (#999) (19b7d13) - fix(import): remove resourceName/executionRoleArn co-variance refine (#996) (ad0ee58) @@ -67,9 +111,13 @@ All notable changes to this project will be documented in this file. - ci: bump the github-actions group across 1 directory with 4 updates (#964) (9962c3e) - test: configure git in browser tests workflow (#976) (17b5727) - fix(import): remove experimental warning from import command (#977) (fdd6631) +- Remove inline container build from vended cdk-stack.ts (#954) (57ee733) - feat(invoke): add --prompt-file and stdin support for long prompts (#974) (f6a3e99) - test: split browser tests into its own job, fix logs path (#975) (acbfb9e) - fix(invoke): auto-generate session ID for bearer-token invocations (#953) (343fedc) +- chore: bump version to 0.11.0 (#967) (f8dc490) +- test: add browser tests for agent inspector (#938) (7a4104d) +- chore: bump version to 0.10.0 (#944) (12275c3) ## [0.11.0] - 2026-04-24 diff --git a/README.md b/README.md index 27dc204ce..af7769bfe 100644 --- a/README.md +++ b/README.md @@ -110,15 +110,29 @@ agentcore invoke ### Evaluations -| Command | Description | -| -------------------- | --------------------------------------------- | -| `add evaluator` | Add a custom LLM-as-a-Judge evaluator | -| `add online-eval` | Add continuous evaluation for live traffic | -| `run eval` | Run on-demand evaluation against agent traces | -| `evals history` | View past eval run results | -| `pause online-eval` | Pause a deployed online eval config | -| `resume online-eval` | Resume a paused online eval config | -| `logs evals` | Stream or search online eval logs | +| Command | Description | +| ----------------------- | ------------------------------------------------ | +| `add evaluator` | Add a custom LLM-as-a-Judge evaluator | +| `add online-eval` | Add continuous evaluation for live traffic | +| `run eval` | Run on-demand evaluation against agent traces | +| `run batch-evaluation` | Run evaluators across all sessions [preview] | +| `run recommendation` | Optimize prompts and tool descriptions [preview] | +| `evals history` | View past eval run results | +| `pause online-eval` | Pause a deployed online eval config | +| `resume online-eval` | Resume a paused online eval config | +| `stop batch-evaluation` | Stop a running batch evaluation [preview] | +| `logs evals` | Stream or search online eval logs | + +### Config Bundles [preview] + +| Command | Description | +| ------------------- | ----------------------------------------- | +| `add config-bundle` | Add a versioned configuration bundle | +| `cb versions` | List version history for a bundle | +| `cb diff` | Diff two versions of a bundle | +| `cb create-branch` | Create a new branch on an existing bundle | + +> Create agents with `--with-config-bundle` to auto-wire config bundle support into the generated template. ### Utilities @@ -171,6 +185,9 @@ Projects use JSON schema files in the `agentcore/` directory: - [CLI Commands Reference](docs/commands.md) - Full command reference for scripting and CI/CD - [Configuration](docs/configuration.md) - Schema reference for config files - [Evaluations](docs/evals.md) - Evaluators, on-demand evals, and online monitoring +- [Batch Evaluation](docs/batch-evaluation.md) - Run evaluators across sessions at scale [preview] +- [Recommendations](docs/recommendations.md) - Optimize prompts and tool descriptions [preview] +- [Config Bundles](docs/config-bundles.md) - Versioned runtime configurations [preview] - [Frameworks](docs/frameworks.md) - Supported frameworks and model providers - [Gateway](docs/gateway.md) - Gateway setup, targets, and authentication - [Memory](docs/memory.md) - Memory strategies and sharing diff --git a/create-bundle.mjs b/create-bundle.mjs new file mode 100644 index 000000000..89bf1d221 --- /dev/null +++ b/create-bundle.mjs @@ -0,0 +1,53 @@ +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { SignatureV4 } from '@aws-sdk/signature-v4'; +import { randomUUID } from 'crypto'; + +const region = 'us-east-1'; +const endpoint = `https://gamma.${region}.elcapcp.genesis-primitives.aws.dev`; + +const body = JSON.stringify({ + bundleName: 'test_rec_bundle', + description: 'Test bundle for recommendation', + clientToken: randomUUID(), + components: { + ['arn:aws:bedrock-agentcore:us-east-1:998846730471:runtime/myproject_MyAgent-QMd093Gl4O']: { + configuration: { + system_prompt: 'You are a helpful assistant that helps users.', + modelId: 'anthropic.claude-sonnet-4-20250514', + }, + }, + }, + branchName: 'mainline', + commitMessage: 'Initial version for rec test', +}); + +const signer = new SignatureV4({ + credentials: defaultProvider(), + region, + service: 'bedrock-agentcore-control', + sha256: Sha256, +}); + +const url = new URL('/configuration-bundles/create', endpoint); +const request = { + method: 'POST', + hostname: url.hostname, + path: url.pathname, + headers: { + 'content-type': 'application/json', + host: url.hostname, + }, + body, +}; + +const signed = await signer.sign(request); +const resp = await fetch(`${endpoint}/configuration-bundles/create`, { + method: 'POST', + headers: signed.headers, + body, +}); + +const data = await resp.text(); +console.log(`Status: ${resp.status}`); +console.log(data); diff --git a/docs/PERMISSIONS.md b/docs/PERMISSIONS.md index 53ed2958d..f041a4b0b 100644 --- a/docs/PERMISSIONS.md +++ b/docs/PERMISSIONS.md @@ -39,8 +39,10 @@ Attach this to every IAM user or role that will run AgentCore CLI commands. The - `sts:GetCallerIdentity`, `cloudformation:DescribeStacks`, `tag:GetResources` for basic operations - `bedrock-agentcore:Invoke*`, `bedrock-agentcore:Get*`, `bedrock-agentcore:List*` for invoking agents and checking status +- Batch evaluation and recommendation actions for `run batch-eval` and `run recommend` - Credential provider and token vault actions for `deploy` when the project uses identity features -- CloudWatch Logs, X-Ray, and Application Signals actions for `logs`, `traces`, and observability setup +- CloudWatch Logs (including log group creation for batch eval results), X-Ray, and Application Signals actions for + `logs`, `traces`, and observability setup - Bedrock actions for agent import and AI-assisted code generation (optional, see [Scoping down by feature](#scoping-down-by-feature)) @@ -162,15 +164,16 @@ The policy files provided cover every AgentCore feature. If your team only uses corresponding statements to further tighten the policies. This table maps features to the policy statements that can be safely removed: -| If your team does not use... | Remove from user policy | Remove from CFN execution policy | -| ------------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| Container builds (CodeZip only) | _(no change)_ | `EcrContainerBuilds`, `CodeBuildContainerBuilds` | -| MCP Lambda compute | _(no change)_ | `LambdaMcpAndCustomResources` (keep if using container builds, which need Lambda for custom resources) | -| Agent import from Bedrock | `BedrockAgentImport` | _(no change)_ | -| AI-assisted code generation | `BedrockModelInvocation` | _(no change)_ | -| Identity/credential providers | `IdentityCredentialManagement`, `TokenVaultKmsKeyCreation` | `SecretsManagerForCredentials` | -| Policy engine | `PolicyGeneration` | Remove `*PolicyEngine*` and `*Policy` actions from `BedrockAgentCoreResources` | -| Online evaluations | Remove `UpdateOnlineEvaluationConfig` from `AgentCoreResourceStatus` | Remove `*OnlineEvaluationConfig*` actions from `BedrockAgentCoreResources` | +| If your team does not use... | Remove from user policy | Remove from CFN execution policy | +| ------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | +| Container builds (CodeZip only) | _(no change)_ | `EcrContainerBuilds`, `CodeBuildContainerBuilds` | +| MCP Lambda compute | _(no change)_ | `LambdaMcpAndCustomResources` (keep if using container builds, which need Lambda for custom resources) | +| Agent import from Bedrock | `BedrockAgentImport` | _(no change)_ | +| AI-assisted code generation | `BedrockModelInvocation` | _(no change)_ | +| Identity/credential providers | `IdentityCredentialManagement`, `TokenVaultKmsKeyCreation` | `SecretsManagerForCredentials` | +| Policy engine | `PolicyGeneration` | Remove `*PolicyEngine*` and `*Policy` actions from `BedrockAgentCoreResources` | +| Online evaluations | Remove `UpdateOnlineEvaluationConfig` from `AgentCoreResourceStatus` | Remove `*OnlineEvaluationConfig*` actions from `BedrockAgentCoreResources` | +| Batch eval / recommendations | `BatchEvalAndRecommendations`; remove `CreateLogGroup`, `CreateLogStream`, `PutLogEvents` from `LogsStreamingAndSearch` | _(no change)_ | ## Hardening with permission boundaries @@ -335,6 +338,20 @@ Required for all deployment operations (`deploy`, `status`, `diff`). | `bedrock-agentcore:Evaluate` | `run evals` | Run on-demand evaluation against agent traces | | `bedrock-agentcore:UpdateOnlineEvaluationConfig` | `pause online-eval`, `resume online-eval` | Pause or resume online evaluation | +### Batch evaluation and recommendations + +| Action | CLI Commands | Purpose | +| ----------------------------------------- | ---------------- | ------------------------------ | +| `bedrock-agentcore:StartBatchEvaluation` | `run batch-eval` | Start a batch evaluation job | +| `bedrock-agentcore:GetBatchEvaluation` | `run batch-eval` | Poll batch evaluation status | +| `bedrock-agentcore:ListBatchEvaluations` | `evals history` | List past batch evaluations | +| `bedrock-agentcore:StopBatchEvaluation` | `run batch-eval` | Stop an in-progress batch eval | +| `bedrock-agentcore:DeleteBatchEvaluation` | `run batch-eval` | Delete a batch evaluation | +| `bedrock-agentcore:StartRecommendation` | `run recommend` | Start a recommendation job | +| `bedrock-agentcore:GetRecommendation` | `run recommend` | Poll recommendation status | +| `bedrock-agentcore:ListRecommendations` | `run recommend` | List past recommendations | +| `bedrock-agentcore:DeleteRecommendation` | `run recommend` | Stop/delete a recommendation | + ### Identity and credential management | Action | CLI Commands | Purpose | @@ -361,14 +378,18 @@ Required for all deployment operations (`deploy`, `status`, `diff`). ### Logging, traces, and observability -| Action | CLI Commands | Purpose | -| ------------------------------- | ---------------------------------------- | --------------------------------------------- | -| `logs:StartLiveTail` | `logs` | Stream agent logs in real-time | -| `logs:FilterLogEvents` | `logs` | Search agent logs | -| `logs:StartQuery` | `traces list`, `traces get`, `run evals` | Run CloudWatch Logs Insights queries | -| `logs:GetQueryResults` | `traces list`, `traces get`, `run evals` | Retrieve query results | -| `logs:DescribeResourcePolicies` | `deploy` | Check for X-Ray log resource policy | -| `logs:PutResourcePolicy` | `deploy` | Create resource policy for X-Ray trace access | +| Action | CLI Commands | Purpose | +| ------------------------------- | ---------------------------------------- | ------------------------------------------------------- | +| `logs:StartLiveTail` | `logs` | Stream agent logs in real-time | +| `logs:FilterLogEvents` | `logs` | Search agent logs | +| `logs:StartQuery` | `traces list`, `traces get`, `run evals` | Run CloudWatch Logs Insights queries | +| `logs:GetQueryResults` | `traces list`, `traces get`, `run evals` | Retrieve query results | +| `logs:DescribeResourcePolicies` | `deploy` | Check for X-Ray log resource policy | +| `logs:PutResourcePolicy` | `deploy` | Create resource policy for X-Ray trace access | +| `logs:DescribeLogGroups` | `run batch-eval`, `run recommend` | Discover runtime log groups for evaluation data sources | +| `logs:CreateLogGroup` | `run batch-eval` | Create log group for batch evaluation results output | +| `logs:CreateLogStream` | `run batch-eval` | Create log stream for batch evaluation results | +| `logs:PutLogEvents` | `run batch-eval` | Write batch evaluation results to CloudWatch Logs | ### Transaction search setup diff --git a/docs/TESTING.md b/docs/TESTING.md index 700ab3aae..9c70af6b3 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -415,6 +415,24 @@ Test configuration is in `vitest.config.ts` using Vitest projects: - Test timeout: 120 seconds - Hook timeout: 120 seconds +## Troubleshooting + +### `Cannot find module '@playwright/test'` + +Playwright is not installed. Run: + +```bash +npm install +``` + +### `browserType.launch: Executable doesn't exist` (Playwright browsers) + +Playwright browsers need to be downloaded after install. Run: + +```bash +npx playwright install chromium +``` + ## Integration Tests Integration tests require: diff --git a/docs/batch-evaluation.md b/docs/batch-evaluation.md new file mode 100644 index 000000000..ea13d3707 --- /dev/null +++ b/docs/batch-evaluation.md @@ -0,0 +1,143 @@ +# Batch Evaluation [preview] + +Batch evaluation runs evaluators across all agent sessions in CloudWatch, producing per-session scores and aggregate +metrics. Use it to measure agent quality over time, compare before/after prompt changes, or validate ground truth +expectations. + +## Quick Start + +```bash +# Run a single evaluator across all sessions +agentcore run batch-evaluation -r MyAgent -e Builtin.Correctness + +# Multiple evaluators +agentcore run batch-evaluation -r MyAgent -e Builtin.Correctness Builtin.Helpfulness Builtin.Faithfulness + +# JSON output for scripting +agentcore run batch-evaluation -r MyAgent -e Builtin.Helpfulness --json +``` + +## Available Evaluators + +Built-in evaluators provided by AgentCore: + +| Evaluator | What it measures | +| ----------------------------------- | ---------------------------------------------- | +| `Builtin.Correctness` | Factual accuracy of responses | +| `Builtin.Helpfulness` | How well responses address the user's goal | +| `Builtin.Faithfulness` | Grounding in tool results / provided context | +| `Builtin.GoalSuccessRate` | Whether the agent achieved the user's goal | +| `Builtin.ToolSelectionAccuracy` | Correct tool chosen for the task | +| `Builtin.Completeness` | Whether all parts of the request were handled | +| `Builtin.TrajectoryExactOrderMatch` | Tool call sequence matches expected trajectory | + +Custom evaluators defined in your project (via `agentcore add evaluator`) can also be used. + +## Filtering Sessions + +### By time window + +```bash +# Only sessions from the last 3 days +agentcore run batch-evaluation -r MyAgent -e Builtin.Helpfulness --lookback-days 3 +``` + +### By session ID + +```bash +agentcore run batch-evaluation -r MyAgent -e Builtin.Correctness -s +``` + +## Ground Truth + +Provide expected responses, assertions, or expected tool trajectories for specific sessions: + +```bash +agentcore run batch-evaluation \ + -r MyAgent \ + -e Builtin.Correctness Builtin.GoalSuccessRate \ + -s \ + --ground-truth ./ground_truth.json +``` + +### Ground truth file format + +```json +[ + { + "sessionId": "", + "groundTruth": { + "inline": { + "assertions": [{ "text": "Agent should use the lookup_order tool" }], + "expectedTrajectory": { + "toolNames": ["lookup_order"] + }, + "turns": [ + { + "input": "What's the status of order ORD-1001?", + "expectedResponse": { "text": "Order ORD-1001 has been delivered" } + } + ] + } + } + } +] +``` + +All fields inside `inline` are optional — include only what's relevant: + +- `assertions` — free-text expectations evaluated by `Builtin.GoalSuccessRate` +- `expectedTrajectory` — tool call sequence evaluated by `Builtin.TrajectoryExactOrderMatch` +- `turns` — input/expected-response pairs evaluated by `Builtin.Correctness` + +## Custom Name + +```bash +agentcore run batch-evaluation -r MyAgent -e Builtin.Helpfulness -n "weekly_quality_check" +``` + +Names must start with a letter and contain only letters, digits, and underscores (max 48 characters). + +## Stopping a Running Evaluation + +```bash +agentcore stop batch-evaluation -i +``` + +## Viewing Results + +### CLI output + +The CLI shows scores grouped by evaluator with average scores after the run completes. + +### Local history + +Results are saved in `.cli/eval-job-results/`. View past runs via the TUI: + +```bash +agentcore +# Navigate to: Evals → Batch Evaluation History +``` + +### JSON output + +```bash +agentcore run batch-evaluation -r MyAgent -e Builtin.Helpfulness --json +``` + +Returns `batchEvaluationId`, `evaluationResults` with `numberOfSessionsCompleted`, `evaluatorSummaries` with +per-evaluator `averageScore`. + +## TUI Wizard + +Run `agentcore` → Run → Batch Evaluation for a guided flow: + +1. Select agent +2. Multi-select evaluators +3. Set lookback days +4. Optionally select specific sessions +5. Optionally add ground truth +6. Name the run (optional) +7. Confirm and run + +The TUI shows real-time progress with elapsed time and step indicators. diff --git a/docs/config-bundles.md b/docs/config-bundles.md new file mode 100644 index 000000000..890ad7aaf --- /dev/null +++ b/docs/config-bundles.md @@ -0,0 +1,114 @@ +# Configuration Bundles [preview] + +Config bundles are versioned configurations that store your agent's runtime settings — system prompt, tool descriptions, +model parameters, or any custom keys. Instead of hardcoding values in your agent code, your agent reads its config at +invocation time from whichever bundle version is active. + +## Concepts + +| Concept | Description | +| ------------- | ----------------------------------------------------------------------------------- | +| **Bundle** | A named container for component configurations, stored in `agentcore.json` | +| **Version** | An immutable snapshot of a bundle's configuration, created on each deploy or update | +| **Branch** | A named lineage within a bundle (e.g. `mainline`, `experiment-1`) | +| **Component** | A runtime or gateway whose configuration is managed by the bundle | + +## Creating a Config Bundle + +### With agent creation + +Create an agent with a pre-wired config bundle that injects system prompt and tool descriptions at runtime: + +```bash +agentcore create --name MyProject --defaults --with-config-bundle +``` + +This creates a `{AgentName}Config` bundle with smart defaults and generates a template that uses +`BedrockAgentCoreContext.get_config_bundle()` to read config at runtime. + +### Standalone + +```bash +agentcore add config-bundle \ + --name MyBundle \ + --description "Production configuration" \ + --components '{"{{runtime:MyAgent}}": {"configuration": {"systemPrompt": "You are helpful.", "temperature": 0.7}}}' \ + --branch mainline \ + --commit-message "Initial config" \ + --json +``` + +The `{{runtime:MyAgent}}` placeholder resolves to the real runtime ARN at deploy time. + +### Via TUI + +Run `agentcore` → Add → select "Configuration Bundle", or select "Config bundle" in the Advanced Configuration step when +adding an agent. + +## Deploying + +```bash +agentcore deploy +``` + +On deploy, the CLI creates or updates the config bundle in the API and stores the bundle ID, ARN, and version ID in +`deployed-state.json`. + +## Managing Versions + +### List versions + +```bash +agentcore cb versions --bundle MyBundle +``` + +Shows version history grouped by branch with commit messages, timestamps, and parent lineage. + +### Diff two versions + +```bash +agentcore cb diff --bundle MyBundle --from --to +``` + +### Create a branch + +```bash +agentcore cb create-branch --bundle MyBundle --branch experiment-1 +``` + +Creates a new branch from the latest version (or a specific version with `--from`). + +## Updating Without Redeploying Code + +Edit the `systemPrompt` or other fields in `agentcore.json` under `configBundles`, then: + +```bash +agentcore deploy +``` + +A new version is created in the API. The next invocation picks up the new config automatically — no code changes needed. + +## How It Works at Runtime + +When you invoke an agent with an associated config bundle, the CLI passes the bundle ARN and version as W3C baggage +headers. The SDK's `BedrockAgentCoreContext.get_config_bundle()` reads the baggage, fetches the config from the API +(cached per version), and makes it available to your agent code. + +The generated template uses a `ConfigBundleHook` (Strands) or `ConfigBundleCallback` (LangGraph) to inject the system +prompt and tool descriptions before each invocation. + +## Bundle Name in agentcore.json + +The CLI prefixes your bundle name with the project name when creating it in the API (e.g. `MyProject` + `MyBundle` → +`MyProjectMyBundle`). You always use the local name (`MyBundle`) in CLI commands — the CLI resolves the prefix +automatically. + +## JSON Output + +All commands support `--json` for scripting: + +```bash +agentcore cb versions --bundle MyBundle --json +agentcore cb diff --bundle MyBundle --from v1 --to v2 --json +agentcore cb create-branch --bundle MyBundle --branch exp-1 --json +``` diff --git a/docs/harness.md b/docs/harness.md new file mode 100644 index 000000000..db51d2b7f --- /dev/null +++ b/docs/harness.md @@ -0,0 +1,263 @@ +# Harness + +A **harness** is a managed agent runtime that connects a foundation model to tools, memory, and configuration — without +requiring you to write agent framework code. You define the model, tools, and settings; AgentCore handles the +orchestration. + +Use a harness when you want a quick, config-driven agent. Use a traditional agent (with `--framework`) when you need +custom code, a specific framework (Strands, LangChain, etc.), or full control over the agent loop. + +## Creating a Harness Project + +```bash +# Minimal — defaults to Bedrock provider, Claude Sonnet +agentcore create --name myharness + +# Specify provider and model +agentcore create --name myharness --model-provider bedrock --model-id global.anthropic.claude-sonnet-4-6 + +# OpenAI provider (requires --api-key-arn) +agentcore create --name myharness --model-provider open_ai --model-id gpt-4o \ + --api-key-arn arn:aws:secretsmanager:us-west-2:123456789012:secret:openai-key + +# Gemini provider +agentcore create --name myharness --model-provider gemini --model-id gemini-2.5-flash \ + --api-key-arn arn:aws:secretsmanager:us-west-2:123456789012:secret:gemini-key + +# Skip auto-created memory +agentcore create --name myharness --no-harness-memory + +# With all optional settings +agentcore create --name myharness \ + --model-provider bedrock \ + --max-iterations 10 \ + --max-tokens 4096 \ + --timeout 120 \ + --truncation-strategy sliding_window \ + --session-storage-mount-path /mnt/data +``` + +### Model Providers + +| Provider | `--model-provider` value | Requires `--api-key-arn` | +| -------- | ------------------------ | ------------------------ | +| Bedrock | `bedrock` | No | +| OpenAI | `open_ai` or `openai` | Yes | +| Gemini | `gemini` | Yes | + +> Aliases `Bedrock`, `OpenAI`, `Gemini`, `Anthropic` (maps to bedrock) are also accepted. + +### Harness vs Agent + +If you pass `--framework`, `--language`, or other agent-specific flags, the CLI creates a traditional agent project +instead. These flags cannot be mixed with harness-only flags (`--model-id`, `--max-iterations`, etc.). + +## Project Structure + +``` +myharness/ + agentcore/ # Config and CDK project + agentcore.json # Project manifest (lists harnesses, memories, etc.) + aws-targets.json # Deployment targets + cdk/ # CDK infrastructure code + app/myharness/ # Harness configuration + harness.json # Harness spec (model, tools, settings) + system-prompt.md # System prompt (editable) +``` + +## Deployment Targets (`aws-targets.json`) + +Before deploying, ensure `aws-targets.json` has at least one target: + +```json +[ + { + "name": "default", + "account": "123456789012", + "region": "us-west-2" + } +] +``` + +Fields: + +- `name` — target name (use `"default"` for the primary target) +- `account` — AWS account ID (string) +- `region` — AWS region + +## Adding a Harness to an Existing Project + +```bash +agentcore add harness --name myharness --model-provider bedrock +agentcore add harness --name myharness --model-provider bedrock --session-storage /mnt/data +agentcore add harness --name myharness --model-provider bedrock --with-invoke-script +``` + +### Custom JWT Auth + +```bash +agentcore add harness --name myharness --model-provider bedrock \ + --authorizer-type CUSTOM_JWT \ + --discovery-url https://example.auth0.com/.well-known/openid-configuration \ + --allowed-audience myapp +``` + +## Tools + +Harnesses support four built-in tool types plus inline functions: + +### Adding Tools + +```bash +# Remote MCP server +agentcore add tool --harness myharness --type remote_mcp --name mytool \ + --url https://mcp-server.example.com/sse + +# Browser tool +agentcore add tool --harness myharness --type agentcore_browser --name browser + +# Code interpreter +agentcore add tool --harness myharness --type agentcore_code_interpreter --name codeinterp + +# Gateway tool (by ARN) +agentcore add tool --harness myharness --type agentcore_gateway --name gwtool \ + --gateway-arn arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/gw-abc + +# Gateway tool (by project gateway name — resolves ARN from deployed state) +agentcore add tool --harness myharness --type agentcore_gateway --name gwtool \ + --gateway mygateway +``` + +### Removing Tools + +```bash +agentcore remove tool --harness myharness --name mytool +``` + +## Session Storage + +Session storage provides a persistent filesystem mount for the harness runtime. Files written to the mount path persist +across invocations within the same session. + +```bash +# Via add harness +agentcore add harness --name myharness --model-provider bedrock --session-storage /mnt/data + +# Via create +agentcore create --name myharness --session-storage-mount-path /mnt/data +``` + +The path must be an absolute path under `/mnt/` (e.g., `/mnt/data`, `/mnt/workspace`). + +**Important:** Only files written to the configured mount path are persistent and visible to `--exec` commands. Files +written to other paths (e.g., `/home`, `/tmp`) may be created in an ephemeral context and will not appear when +inspecting the container via `--exec`. If your tools write files, configure them to use the session storage path. + +## Deploying + +```bash +agentcore deploy # Interactive — prompts for confirmation +agentcore deploy -y # Auto-confirm +agentcore deploy --dry-run # Preview without deploying +agentcore deploy --diff # Show CDK diff +``` + +Deploy creates: + +1. CloudFormation stack (IAM role, memory) +2. Harness resource via AgentCore API + +## Checking Status + +```bash +agentcore status # All resources +agentcore status --type harness # Harness resources only +agentcore status --json # JSON output +``` + +## Invoking + +```bash +# Basic invoke +agentcore invoke --harness myharness "What can you do?" + +# With session continuity +agentcore invoke --harness myharness --session-id "Follow up question" + +# Verbose — shows raw streaming events +agentcore invoke --harness myharness --verbose "Hello" + +# JSON output +agentcore invoke --harness myharness --json "Hello" +``` + +### Invoke Overrides + +These flags override harness settings for a single invocation only (they do not persist): + +| Flag | Description | +| ----------------------------- | ------------------------------------- | +| `--model-id ` | Use a different model | +| `--system-prompt ` | Override the system prompt | +| `--max-iterations ` | Override max agent loop iterations | +| `--max-tokens ` | Override max tokens per iteration | +| `--harness-timeout ` | Override execution timeout | +| `--tools ` | Override tools (comma-separated) | +| `--allowed-tools ` | Restrict which tools can be used | +| `--skills ` | Skills to use (comma-separated paths) | +| `--actor-id ` | Override memory actor ID | +| `--bearer-token ` | Bearer token for CUSTOM_JWT auth | + +## Logs and Traces + +```bash +# View logs +agentcore logs --harness myharness --limit 20 +agentcore logs --harness myharness --since 1h --level error + +# List traces +agentcore traces list --harness myharness +agentcore traces list --harness myharness --since 30m --limit 10 + +# Download a trace +agentcore traces get --harness myharness +agentcore traces get --harness myharness --output ./trace.json +``` + +## Fetching Access Info + +For harnesses with CUSTOM_JWT auth: + +```bash +agentcore fetch access --name myharness --type harness +agentcore fetch access --name myharness --type harness --json +``` + +## Removing a Harness + +```bash +agentcore remove harness --name myharness -y +agentcore deploy # Apply removal to AWS +``` + +## Validating Configuration + +```bash +agentcore validate +``` + +Checks: + +- Harness schema validity (model, tools, settings) +- Cross-references (memory names exist in project) +- Tool configuration completeness + +## Invoke Script + +Pass `--with-invoke-script` to generate a standalone Python script for invoking the harness outside the CLI: + +```bash +agentcore add harness --name myharness --model-provider bedrock --with-invoke-script +``` + +This creates `app/myharness/invoke.py` which uses `boto3` to invoke the harness directly. diff --git a/docs/policies/iam-policy-user.json b/docs/policies/iam-policy-user.json index d2467a134..8e268824f 100644 --- a/docs/policies/iam-policy-user.json +++ b/docs/policies/iam-policy-user.json @@ -87,6 +87,22 @@ "Action": ["kms:CreateKey", "kms:TagResource"], "Resource": "*" }, + { + "Sid": "BatchEvalAndRecommendations", + "Effect": "Allow", + "Action": [ + "bedrock-agentcore:StartBatchEvaluation", + "bedrock-agentcore:GetBatchEvaluation", + "bedrock-agentcore:ListBatchEvaluations", + "bedrock-agentcore:StopBatchEvaluation", + "bedrock-agentcore:DeleteBatchEvaluation", + "bedrock-agentcore:StartRecommendation", + "bedrock-agentcore:GetRecommendation", + "bedrock-agentcore:ListRecommendations", + "bedrock-agentcore:DeleteRecommendation" + ], + "Resource": "*" + }, { "Sid": "LogsStreamingAndSearch", "Effect": "Allow", @@ -96,7 +112,11 @@ "logs:StartQuery", "logs:GetQueryResults", "logs:DescribeResourcePolicies", - "logs:PutResourcePolicy" + "logs:PutResourcePolicy", + "logs:DescribeLogGroups", + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" ], "Resource": "*" }, diff --git a/docs/recommendations.md b/docs/recommendations.md new file mode 100644 index 000000000..c5a5c4ac3 --- /dev/null +++ b/docs/recommendations.md @@ -0,0 +1,158 @@ +# Recommendations [preview] + +Recommendations optimize your agent's system prompt or tool descriptions using historical traces as signal. The +recommendation service analyzes how your agent performed, then produces an improved version scored by an evaluator. + +## Quick Start + +```bash +# Optimize a system prompt (inline) +agentcore run recommendation \ + -r MyAgent \ + -e Builtin.Helpfulness \ + --type system-prompt \ + --inline "You are a helpful assistant." + +# Optimize tool descriptions +agentcore run recommendation \ + -r MyAgent \ + --type tool-description \ + --tools "search:Searches the web" "calc:Does math" +``` + +## System Prompt Recommendations + +### From inline text + +```bash +agentcore run recommendation \ + -r MyAgent \ + -e Builtin.Helpfulness \ + --type system-prompt \ + --inline "You are a helpful assistant. Use tools when appropriate." +``` + +### From a file + +```bash +agentcore run recommendation \ + -r MyAgent \ + -e Builtin.Helpfulness \ + --type system-prompt \ + --prompt-file ./system-prompt.txt +``` + +### From a config bundle + +Read the current prompt from a deployed config bundle, optimize it, and write the result back as a new bundle version: + +```bash +agentcore run recommendation \ + -r MyAgent \ + -e Builtin.Helpfulness \ + --type system-prompt \ + --bundle-name MyBundle \ + --bundle-version \ + --system-prompt-json-path systemPrompt +``` + +The `--system-prompt-json-path` is the field name under `configuration` in the bundle (e.g. `systemPrompt`). The CLI +resolves it to the full path automatically using the component ARN from your deployed state. + +> **JSONPath format:** The API uses dot notation (`$.{ARN}.configuration.{field}`), not standard JSONPath bracket +> notation. You don't need to worry about this — just pass the short field name and the CLI handles the resolution. If +> you need the full path for direct API calls, use `$.arn:aws:...:runtime/MyAgent.configuration.systemPrompt` (no +> brackets, no quotes around the ARN). + +On success, the recommendation writes a new config bundle version with the optimized prompt. The agent picks it up on +the next invocation — no redeploy needed. + +## Tool Description Recommendations + +```bash +agentcore run recommendation \ + -r MyAgent \ + --type tool-description \ + --tools "add_numbers:Return the sum of two numbers" "search:Searches the web" +``` + +Returns optimized tool descriptions for each tool. + +## Trace Source + +By default, the recommendation service fetches traces from CloudWatch using a 7-day lookback. Customize with: + +```bash +# Custom lookback window +agentcore run recommendation ... --lookback 14 + +# Specific sessions only +agentcore run recommendation ... --session-id + +# From a local spans file (OTEL format) +agentcore run recommendation ... --spans-file ./traces.json +``` + +## JSON Output + +```bash +agentcore run recommendation -r MyAgent -e Builtin.Helpfulness --type system-prompt --inline "..." --json +``` + +Returns `recommendationId`, `status`, and `result` with `systemPromptRecommendationResult.recommendedSystemPrompt` or +`toolDescriptionRecommendationResult.tools`. + +When using `--bundle-name`, the result also includes `configurationBundle.versionId` — the new bundle version. + +## End-to-End Workflow: Recommendation → Config Bundle → Invoke + +1. Create agent with config bundle: + + ```bash + agentcore create --name MyAgent --defaults --with-config-bundle + agentcore deploy + ``` + +2. Invoke a few times to generate traces: + + ```bash + agentcore invoke --prompt "What is 2 + 3?" + agentcore invoke --prompt "Tell me about Paris" + ``` + +3. Run recommendation from config bundle: + + ```bash + agentcore run recommendation \ + -r MyAgent -e Builtin.Helpfulness --type system-prompt \ + --bundle-name MyAgentConfig --bundle-version \ + --system-prompt-json-path systemPrompt + ``` + +4. Invoke again — the agent uses the optimized prompt without code changes: + ```bash + agentcore invoke --prompt "Who are you?" + ``` + +## Viewing History + +Results are saved in `.cli/recommendations/`. View past runs via the TUI: + +```bash +agentcore +# Navigate to: Recommendations → History +``` + +## TUI Wizard + +Run `agentcore` → Run → Recommendation for a guided flow: + +1. Select recommendation type (system prompt or tool description) +2. Select agent +3. Select evaluator (system prompt only) +4. Choose input source (inline, file, or config bundle) +5. Choose trace source (CloudWatch or sessions) +6. Confirm and run + +The TUI shows real-time progress and displays the recommended changes when complete, with an option to apply config +bundle updates. diff --git a/e2e-tests/ab-test-config-bundle.test.ts b/e2e-tests/ab-test-config-bundle.test.ts new file mode 100644 index 000000000..9c18b2f31 --- /dev/null +++ b/e2e-tests/ab-test-config-bundle.test.ts @@ -0,0 +1,211 @@ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: config-bundle AB test lifecycle', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2eCfgAB${String(Date.now()).slice(-8)}`; + const abTestName = 'ConfigBundleABTest'; + const evalName = 'BundleEvaluator'; + const onlineEvalName = 'BundleOnlineEval'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-cfg-ab-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + it.skipIf(!canRun)( + 'adds evaluator and online eval config', + async () => { + let result = await run([ + 'add', + 'evaluator', + '--name', + evalName, + '--level', + 'SESSION', + '--model', + 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + '--instructions', + 'Evaluate session quality. Context: {context}', + '--json', + ]); + expect(result.exitCode, `Add evaluator failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'online-eval', + '--name', + onlineEvalName, + '--runtime', + agentName, + '--evaluator', + evalName, + '--sampling-rate', + '100', + '--enable-on-create', + '--json', + ]); + expect(result.exitCode, `Add online-eval failed: ${result.stdout}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'deploys agent before AB test (needed for config bundles)', + async () => { + await retry( + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Initial deploy stdout:', result.stdout); + console.log('Initial deploy stderr:', result.stderr); + } + expect(result.exitCode, `Initial deploy failed`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 2, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'adds config-bundle AB test with 90/10 split', + async () => { + // Config bundles reference ARNs from deployed resources. + // Use placeholder bundle ARNs — the deploy step will validate or create them. + const controlBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/control-v1`; + const treatmentBundle = `arn:aws:bedrock-agentcore:ap-southeast-2:998846730471:config-bundle/treatment-v1`; + + const result = await run([ + 'add', + 'ab-test', + '--mode', + 'config-bundle', + '--name', + abTestName, + '--runtime', + agentName, + '--control-bundle', + controlBundle, + '--control-version', + 'v1', + '--treatment-bundle', + treatmentBundle, + '--treatment-version', + 'v1', + '--control-weight', + '90', + '--treatment-weight', + '10', + '--online-eval', + onlineEvalName, + '--json', + ]); + expect(result.exitCode, `Add AB test failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; abTestName: string }; + expect(json.success).toBe(true); + expect(json.abTestName).toBe(abTestName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'status shows AB test in config', + async () => { + const result = await run(['status', '--json']); + expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string }[]; + }; + expect(json.success).toBe(true); + + // Agent should be deployed + const agent = json.resources.find(r => r.resourceType === 'agent' && r.name === agentName); + expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined(); + expect(agent!.deploymentState).toBe('deployed'); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed agent', + async () => { + await retry( + async () => { + const result = await run(['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json']); + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'removes config-bundle AB test', + async () => { + const result = await run(['remove', 'ab-test', '--name', abTestName, '--json']); + expect(result.exitCode, `Remove failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + }, + 60000 + ); +}); diff --git a/e2e-tests/ab-test-target-based.test.ts b/e2e-tests/ab-test-target-based.test.ts new file mode 100644 index 000000000..ac687e4fb --- /dev/null +++ b/e2e-tests/ab-test-target-based.test.ts @@ -0,0 +1,301 @@ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: target-based AB test lifecycle', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2eTargAB${String(Date.now()).slice(-8)}`; + const abTestName = 'TargetABTest'; + const evalName = 'ABTestEvaluator'; + const controlEvalName = 'ControlEvalConfig'; + const treatmentEvalName = 'TreatmentEvalConfig'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-target-ab-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + it.skipIf(!canRun)( + 'adds runtime endpoints (prod v1, staging v1)', + async () => { + let result = await run([ + 'add', + 'runtime-endpoint', + '--runtime', + agentName, + '--endpoint', + 'prod', + '--version', + '1', + '--json', + ]); + expect(result.exitCode, `Add prod endpoint failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'runtime-endpoint', + '--runtime', + agentName, + '--endpoint', + 'staging', + '--version', + '1', + '--json', + ]); + expect(result.exitCode, `Add staging endpoint failed: ${result.stdout}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds evaluator and per-variant online eval configs', + async () => { + let result = await run([ + 'add', + 'evaluator', + '--name', + evalName, + '--level', + 'SESSION', + '--model', + 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + '--instructions', + 'Evaluate quality. Context: {context}', + '--json', + ]); + expect(result.exitCode, `Add evaluator failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'online-eval', + '--name', + controlEvalName, + '--runtime', + agentName, + '--evaluator', + evalName, + '--sampling-rate', + '100', + '--endpoint', + 'prod', + '--enable-on-create', + '--json', + ]); + expect(result.exitCode, `Add control online-eval failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'online-eval', + '--name', + treatmentEvalName, + '--runtime', + agentName, + '--evaluator', + evalName, + '--sampling-rate', + '100', + '--endpoint', + 'staging', + '--enable-on-create', + '--json', + ]); + expect(result.exitCode, `Add treatment online-eval failed: ${result.stdout}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds target-based AB test with 90/10 split', + async () => { + const result = await run([ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + abTestName, + '--runtime', + agentName, + '--gateway', + `${abTestName}-gw`, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '90', + '--treatment-weight', + '10', + '--control-online-eval', + controlEvalName, + '--treatment-online-eval', + treatmentEvalName, + '--enable', + '--json', + ]); + expect(result.exitCode, `Add AB test failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean; abTestName: string }; + expect(json.success).toBe(true); + expect(json.abTestName).toBe(abTestName); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'deploys project (creates gateway, targets, AB test, eval configs)', + async () => { + await retry( + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 2, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'status shows all resources deployed', + async () => { + await retry( + async () => { + const result = await run(['status', '--json']); + expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string }[]; + }; + expect(json.success).toBe(true); + + // Agent should be deployed + const agent = json.resources.find(r => r.resourceType === 'agent' && r.name === agentName); + expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined(); + expect(agent!.deploymentState).toBe('deployed'); + + // Gateway should be deployed + const gateway = json.resources.find(r => r.resourceType === 'http-gateway' && r.name === `${abTestName}-gw`); + expect(gateway, 'HTTP gateway should appear in status').toBeDefined(); + }, + 3, + 15000 + ); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'pauses AB test', + async () => { + await retry( + async () => { + const result = await run(['pause', 'ab-test', abTestName, '--json']); + expect(result.exitCode, `Pause failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('executionStatus', 'PAUSED'); + }, + 3, + 10000 + ); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'resumes AB test', + async () => { + await retry( + async () => { + const result = await run(['resume', 'ab-test', abTestName, '--json']); + expect(result.exitCode, `Resume failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('executionStatus', 'RUNNING'); + }, + 3, + 10000 + ); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'promotes AB test (updates agentcore.json)', + async () => { + const result = await run(['promote', 'ab-test', abTestName, '--json']); + expect(result.exitCode, `Promote failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('promoted', true); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'removes AB test from config', + async () => { + const result = await run(['remove', 'ab-test', '--name', abTestName, '--delete-gateway', '--json']); + expect(result.exitCode, `Remove failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + }, + 60000 + ); +}); diff --git a/e2e-tests/byo-custom-jwt.test.ts b/e2e-tests/byo-custom-jwt.test.ts index b7391a522..64e534e20 100644 --- a/e2e-tests/byo-custom-jwt.test.ts +++ b/e2e-tests/byo-custom-jwt.test.ts @@ -48,7 +48,7 @@ const region = process.env.AWS_REGION ?? 'us-east-1'; * Run the local CLI build without skipping install (needed for deploy). */ function runLocalCLI(args: string[], cwd: string): Promise { - return runCLI(args, cwd, /* skipInstall */ false); + return runCLI(args, cwd, { skipInstall: false }); } describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => { diff --git a/e2e-tests/config-bundle-eval-rec.test.ts b/e2e-tests/config-bundle-eval-rec.test.ts new file mode 100644 index 000000000..fcc95c5c7 --- /dev/null +++ b/e2e-tests/config-bundle-eval-rec.test.ts @@ -0,0 +1,633 @@ +/** + * E2E tests for Config Bundles, Batch Evaluation, and Recommendations. + * + * Flow: create project → add config bundle → add evaluator → deploy → + * invoke → test config-bundle CLI → run batch-evaluation → run recommendation + * + * Prerequisites: + * - AWS credentials + * - npm, git, uv installed + */ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: config bundles, batch evaluation, and recommendations', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2eCbEr${String(Date.now()).slice(-8)}`; + const bundleName = 'E2eTestBundle'; + const evalName = 'E2eCustomEval'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-cb-eval-rec-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + // Create project with agent + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + // ════════════════════════════════════════════════════════════════════════ + // Config Bundle — add to project + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'adds a config bundle to the project', + async () => { + const components = JSON.stringify({ + [`{{runtime:${agentName}}}`]: { + configuration: { + systemPrompt: 'You are a helpful e2e test assistant.', + temperature: 0.7, + }, + }, + }); + + const result = await run([ + 'add', + 'config-bundle', + '--name', + bundleName, + '--description', + 'E2E test config bundle', + '--components', + components, + '--branch', + 'mainline', + '--commit-message', + 'Initial e2e bundle', + '--json', + ]); + + expect(result.exitCode, `Add config-bundle failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(true); + expect(json.bundleName).toBe(bundleName); + }, + 60000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Evaluator — add to project + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'adds a custom evaluator to the project', + async () => { + const result = await run([ + 'add', + 'evaluator', + '--name', + evalName, + '--level', + 'SESSION', + '--model', + 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + '--instructions', + 'Evaluate the overall quality of this session. Context: {context}', + '--json', + ]); + + expect(result.exitCode, `Add evaluator failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(true); + expect(json.evaluatorName).toBe(evalName); + }, + 60000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Deploy + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'deploys the project with config bundle and evaluator', + async () => { + const result = await run(['deploy', '--yes', '--json']); + + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + + expect(result.exitCode, 'Deploy failed').toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 600000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Invoke — generate traces for evaluation + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'invokes the deployed agent to generate traces', + async () => { + await retry( + async () => { + const result = await run(['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json']); + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Status — verify config bundle and evaluator deployed + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'status shows deployed config bundle and evaluator', + async () => { + const result = await run(['status', '--json']); + + expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string }[]; + }; + expect(json.success).toBe(true); + + const bundle = json.resources.find(r => r.resourceType === 'config-bundle' && r.name === bundleName); + expect(bundle, `Config bundle "${bundleName}" should appear in status`).toBeDefined(); + + const evaluator = json.resources.find(r => r.resourceType === 'evaluator' && r.name === evalName); + expect(evaluator, `Evaluator "${evalName}" should appear in status`).toBeDefined(); + }, + 120000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Config Bundle — versions and diff via CLI + // ════════════════════════════════════════════════════════════════════════ + + let initialVersionId: string; + + it.skipIf(!canRun)( + 'config-bundle versions lists the deployed version', + async () => { + const result = await run(['config-bundle', 'versions', '--bundle', bundleName, '--json']); + + expect(result.exitCode, `cb versions failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + versions: { versionId: string; lineageMetadata?: { branchName?: string; commitMessage?: string } }[]; + bundleName: string; + }; + + expect(json.bundleName).toBe(bundleName); + expect(json.versions.length).toBeGreaterThanOrEqual(1); + initialVersionId = json.versions[0]!.versionId; + expect(initialVersionId).toBeTruthy(); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'config-bundle versions supports --branch filter', + async () => { + const result = await run(['config-bundle', 'versions', '--bundle', bundleName, '--branch', 'mainline', '--json']); + + expect(result.exitCode, `cb versions --branch failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + versions: { versionId: string; lineageMetadata?: { branchName?: string } }[]; + }; + + for (const v of json.versions) { + expect(v.lineageMetadata?.branchName).toBe('mainline'); + } + }, + 120000 + ); + + it.skipIf(!canRun)( + 'updates config bundle by redeploying with changed components', + async () => { + // Update the config bundle in agentcore.json with new component values + const components = JSON.stringify({ + [`{{runtime:${agentName}}}`]: { + configuration: { + systemPrompt: 'You are an UPDATED e2e test assistant.', + temperature: 0.9, + maxTokens: 2048, + }, + }, + }); + + // Remove old bundle, add new one with same name but different components + let result = await run(['remove', 'config-bundle', '--name', bundleName, '--json']); + expect(result.exitCode, `Remove config-bundle failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'config-bundle', + '--name', + bundleName, + '--description', + 'E2E test config bundle - updated', + '--components', + components, + '--branch', + 'mainline', + '--commit-message', + 'Update system prompt and add maxTokens', + '--json', + ]); + expect(result.exitCode, `Re-add config-bundle failed: ${result.stdout}`).toBe(0); + + // Redeploy to push the updated bundle + result = await run(['deploy', '--yes', '--json']); + expect(result.exitCode, `Redeploy failed: ${result.stdout}`).toBe(0); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'config-bundle versions shows both versions after update', + async () => { + const result = await run(['config-bundle', 'versions', '--bundle', bundleName, '--json']); + + expect(result.exitCode, `cb versions failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { + versions: { versionId: string }[]; + }; + + expect(json.versions.length).toBeGreaterThanOrEqual(2); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'config-bundle diff shows changes between versions', + async () => { + // Get the latest two versions + const versionsResult = await run(['config-bundle', 'versions', '--bundle', bundleName, '--json']); + const versionsJson = parseJsonOutput(versionsResult.stdout) as { + versions: { versionId: string }[]; + }; + + expect(versionsJson.versions.length).toBeGreaterThanOrEqual(2); + const newestVersion = versionsJson.versions[0]!.versionId; + const oldestVersion = versionsJson.versions[versionsJson.versions.length - 1]!.versionId; + + const result = await run([ + 'config-bundle', + 'diff', + '--bundle', + bundleName, + '--from', + oldestVersion, + '--to', + newestVersion, + '--json', + ]); + + expect(result.exitCode, `cb diff failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('fromVersion'); + expect(json).toHaveProperty('toVersion'); + expect(json.diffs).toBeInstanceOf(Array); + expect((json.diffs as unknown[]).length).toBeGreaterThan(0); + }, + 120000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Batch Evaluation — run through CLI + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'runs batch evaluation with Builtin evaluator via CLI', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'batch-evaluation', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--lookback-days', + '1', + '--json', + ]); + + expect(result.exitCode, `batch-evaluation failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe( + 0 + ); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('batchEvaluateId'); + expect(json.status).toBeDefined(); + expect(json.status).not.toBe('FAILED'); + }, + 6, + 15000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'runs batch evaluation with ground truth file', + async () => { + // Invoke to get a real session ID for ground truth + const invokeResult = await run(['invoke', '--prompt', 'What is 2+2?', '--runtime', agentName, '--json']); + expect(invokeResult.exitCode).toBe(0); + const invokeJson = parseJsonOutput(invokeResult.stdout) as { sessionId: string }; + expect(invokeJson.sessionId).toBeTruthy(); + + // Create ground truth file using the real session ID + const gtData = [ + { + sessionId: invokeJson.sessionId, + groundTruth: { + inline: { + assertions: [{ text: 'Agent should provide a numerical answer' }], + }, + }, + }, + ]; + const gtPath = join(projectPath, 'ground-truth.json'); + await writeFile(gtPath, JSON.stringify(gtData)); + + await retry( + async () => { + const result = await run([ + 'run', + 'batch-evaluation', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Correctness', + '--ground-truth', + gtPath, + '--lookback-days', + '1', + '--json', + ]); + + expect(result.exitCode, `batch-evaluation with GT failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + }, + 6, + 15000 + ); + }, + 600000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // On-demand Eval — run eval via CLI (existing pattern) + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'runs on-demand eval with Builtin evaluator via CLI', + async () => { + // Retries needed: traces from invoke take time to propagate to CloudWatch + await retry( + async () => { + const result = await run([ + 'run', + 'eval', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--lookback', + '1', + '--json', + ]); + + expect(result.exitCode, `run eval failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('run'); + expect(json).toHaveProperty('filePath'); + }, + 10, + 15000 + ); + }, + 300000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Recommendation — run through CLI + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'runs system prompt recommendation with inline content via CLI', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'recommendation', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant for testing.', + '--lookback', + '1', + '--json', + ]); + + expect(result.exitCode, `recommendation failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('recommendationId'); + expect(json.result).toBeDefined(); + expect(json.result).not.toBe(''); + expect(json.result).not.toBeNull(); + }, + 6, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'runs system prompt recommendation with prompt file via CLI', + async () => { + const promptFile = join(projectPath, 'system-prompt.txt'); + await writeFile(promptFile, 'You are a helpful customer support assistant. Answer politely.'); + + await retry( + async () => { + const result = await run([ + 'run', + 'recommendation', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Helpfulness', + '--prompt-file', + promptFile, + '--lookback', + '1', + '--json', + ]); + + expect(result.exitCode, `recommendation from file failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('recommendationId'); + }, + 6, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'runs tool description recommendation via CLI', + async () => { + await retry( + async () => { + const result = await run([ + 'run', + 'recommendation', + '--type', + 'tool-description', + '--runtime', + agentName, + '--tools', + 'search:Searches the web for information', + '--tools', + 'calculator:Performs mathematical calculations', + '--lookback', + '1', + '--json', + ]); + + expect(result.exitCode, `tool-desc recommendation failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('recommendationId'); + }, + 6, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'runs recommendation with config bundle source via CLI', + async () => { + // Get the latest version ID for the bundle + const versionsResult = await run(['config-bundle', 'versions', '--bundle', bundleName, '--json']); + const versionsJson = parseJsonOutput(versionsResult.stdout) as { + versions: { versionId: string }[]; + }; + const latestVersion = versionsJson.versions[0]!.versionId; + + await retry( + async () => { + const result = await run([ + 'run', + 'recommendation', + '--runtime', + agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--bundle-name', + bundleName, + '--bundle-version', + latestVersion, + '--system-prompt-json-path', + 'systemPrompt', + '--lookback', + '1', + '--json', + ]); + + expect(result.exitCode, `bundle recommendation failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as Record; + expect(json).toHaveProperty('success', true); + expect(json).toHaveProperty('recommendationId'); + }, + 6, + 30000 + ); + }, + 600000 + ); + + // ════════════════════════════════════════════════════════════════════════ + // Cleanup — remove config bundle from project + // ════════════════════════════════════════════════════════════════════════ + + it.skipIf(!canRun)( + 'removes config bundle from project and redeploys (reconciliation deletes it)', + async () => { + let result = await run(['remove', 'config-bundle', '--name', bundleName, '--json']); + expect(result.exitCode, `Remove config-bundle failed: ${result.stdout}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(true); + + // Redeploy triggers reconciliation (orphaned bundle deleted server-side) + result = await run(['deploy', '--yes', '--json']); + expect(result.exitCode, `Final deploy failed: ${result.stdout}`).toBe(0); + }, + 600000 + ); +}); diff --git a/e2e-tests/fixtures/import/cleanup_resources.py b/e2e-tests/fixtures/import/cleanup_resources.py index 0728b711e..120ced18f 100644 --- a/e2e-tests/fixtures/import/cleanup_resources.py +++ b/e2e-tests/fixtures/import/cleanup_resources.py @@ -51,10 +51,6 @@ def main(): rid = val.get("id") if not rid: continue - # Gateway targets are deleted automatically when the parent gateway is deleted - if "gateway" in key and "target" in key: - print(f"Skipping {key} (deleted with parent gateway)") - continue try: if "runtime" in key: client.delete_agent_runtime(agentRuntimeId=rid) @@ -62,8 +58,6 @@ def main(): client.delete_memory(memoryId=rid) elif "evaluator" in key: client.delete_evaluator(evaluatorId=rid) - elif "gateway" in key: - client.delete_gateway(gatewayIdentifier=rid) print(f"Deleted {key}: {rid}") except Exception as e: print(f"Could not delete {key} ({rid}): {e}") diff --git a/e2e-tests/fixtures/import/common.py b/e2e-tests/fixtures/import/common.py index 369ec0bb0..3573ed519 100644 --- a/e2e-tests/fixtures/import/common.py +++ b/e2e-tests/fixtures/import/common.py @@ -2,15 +2,20 @@ import json import os import time +import uuid import zipfile import tempfile import boto3 REGION = os.environ.get("AWS_REGION") or os.environ.get("AWS_DEFAULT_REGION") or "us-east-1" +RESOURCE_SUFFIX = os.environ.get("RESOURCE_SUFFIX", "") +# Unique suffix for resource names — avoids collisions across parallel CI shards. +NAME_SUFFIX = RESOURCE_SUFFIX or uuid.uuid4().hex[:12] SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) APP_DIR = os.path.join(SCRIPT_DIR, "app") -RESOURCES_FILE = os.path.join(SCRIPT_DIR, "bugbash-resources.json") +_resources_name = f"bugbash-resources-{RESOURCE_SUFFIX}.json" if RESOURCE_SUFFIX else "bugbash-resources.json" +RESOURCES_FILE = os.path.join(SCRIPT_DIR, _resources_name) INLINE_POLICY_NAME = "bugbash-agentcore-permissions" @@ -35,6 +40,8 @@ def upload_code(prefix="bugbash"): """Zip APP_DIR and upload to S3. Returns (bucket, s3_key).""" bucket_name = get_code_bucket() s3 = boto3.client("s3", region_name=REGION) + if RESOURCE_SUFFIX: + prefix = f"{prefix}-{RESOURCE_SUFFIX}" # Create zip of app directory with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as tmp: diff --git a/e2e-tests/fixtures/import/setup_evaluator.py b/e2e-tests/fixtures/import/setup_evaluator.py index d49787d0e..e4573da45 100644 --- a/e2e-tests/fixtures/import/setup_evaluator.py +++ b/e2e-tests/fixtures/import/setup_evaluator.py @@ -8,10 +8,10 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( get_control_client, save_resource, tag_resource, wait_for_evaluator, print_import_command, + NAME_SUFFIX, ) DEFAULT_EVALUATOR_MODEL = os.environ.get("DEFAULT_EVALUATOR_MODEL", "us.anthropic.claude-sonnet-4-5-20250929-v1:0") @@ -19,8 +19,7 @@ def main(): client = get_control_client() - ts = int(time.time()) - evaluator_name = f"bugbash_eval_{ts}" + evaluator_name = f"bugbash_eval_{NAME_SUFFIX}" print(f"Creating evaluator: {evaluator_name}") resp = client.create_evaluator( diff --git a/e2e-tests/fixtures/import/setup_gateway.py b/e2e-tests/fixtures/import/setup_gateway.py index e190d0dfc..a846617aa 100644 --- a/e2e-tests/fixtures/import/setup_gateway.py +++ b/e2e-tests/fixtures/import/setup_gateway.py @@ -12,18 +12,17 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( REGION, get_control_client, ensure_role, save_resource, tag_resource, wait_for_gateway, wait_for_gateway_target, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - ts = int(time.time()) - gateway_name = f"bugbashGw{ts}" + gateway_name = f"bugbashGw{NAME_SUFFIX}" # ------------------------------------------------------------------ # 1. Create gateway diff --git a/e2e-tests/fixtures/import/setup_memory_full.py b/e2e-tests/fixtures/import/setup_memory_full.py index 5df196524..277179cfb 100644 --- a/e2e-tests/fixtures/import/setup_memory_full.py +++ b/e2e-tests/fixtures/import/setup_memory_full.py @@ -8,22 +8,22 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( ensure_role, get_control_client, wait_for_memory, save_resource, print_import_command, tag_resource, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - memory_name = f"bugbash_memory_{int(time.time())}" + memory_name = f"bugbash_memory_{NAME_SUFFIX}" print(f"Creating memory: {memory_name}") resp = client.create_memory( name=memory_name, - clientToken=f"bugbash-{int(time.time())}", + clientToken=f"bugbash-{NAME_SUFFIX}", eventExpiryDuration=30, memoryExecutionRoleArn=role_arn, memoryStrategies=[ diff --git a/e2e-tests/fixtures/import/setup_runtime_basic.py b/e2e-tests/fixtures/import/setup_runtime_basic.py index 65e1585a1..d29ddbd1c 100644 --- a/e2e-tests/fixtures/import/setup_runtime_basic.py +++ b/e2e-tests/fixtures/import/setup_runtime_basic.py @@ -7,20 +7,19 @@ import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import time from common import ( ensure_role, get_control_client, wait_for_runtime, save_resource, print_import_command, upload_code, + NAME_SUFFIX, ) def main(): role_arn = ensure_role() client = get_control_client() - ts = int(time.time()) - runtime_name = f"bugbash_basic_{ts}" + runtime_name = f"bugbash_basic_{NAME_SUFFIX}" - bucket, s3_key = upload_code(f"bugbash-basic-{ts}") + bucket, s3_key = upload_code(f"bugbash-basic-{NAME_SUFFIX}") print(f"Creating basic runtime: {runtime_name}") resp = client.create_agent_runtime( diff --git a/e2e-tests/harness-bedrock.test.ts b/e2e-tests/harness-bedrock.test.ts new file mode 100644 index 000000000..7b53e18bb --- /dev/null +++ b/e2e-tests/harness-bedrock.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'bedrock' }); diff --git a/e2e-tests/harness-e2e-helper.ts b/e2e-tests/harness-e2e-helper.ts new file mode 100644 index 000000000..ca29ae4f3 --- /dev/null +++ b/e2e-tests/harness-e2e-helper.ts @@ -0,0 +1,163 @@ +import { hasAwsCredentials, parseJsonOutput, prereqs, retry, spawnAndCollect } from '../src/test-utils/index.js'; +import { + cleanupStaleCredentialProviders, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const hasAws = hasAwsCredentials(); +const baseCanRun = prereqs.npm && prereqs.git && hasAws; + +interface HarnessE2EConfig { + modelProvider: 'bedrock' | 'open_ai' | 'gemini'; + requiredEnvVar?: string; + skipMemory?: boolean; +} + +export function createHarnessE2ESuite(cfg: HarnessE2EConfig) { + const hasRequiredVar = !cfg.requiredEnvVar || !!process.env[cfg.requiredEnvVar]; + const canRun = baseCanRun && hasRequiredVar; + + const providerLabel = + cfg.modelProvider === 'open_ai' ? 'OpenAI' : cfg.modelProvider === 'gemini' ? 'Gemini' : 'Bedrock'; + + describe.sequential(`e2e: harness/${providerLabel} — create → deploy → invoke`, () => { + let testDir: string; + let projectPath: string; + let harnessName: string; + + beforeAll(async () => { + if (!canRun) return; + + await cleanupStaleCredentialProviders(); + + testDir = join(tmpdir(), `agentcore-e2e-harness-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const providerSlug = cfg.modelProvider.replace('_', '').slice(0, 4); + harnessName = `E2eHrns${providerSlug}${String(Date.now()).slice(-8)}`; + + const createArgs = [ + 'create', + '--name', + harnessName, + '--model-provider', + cfg.modelProvider, + '--json', + '--skip-git', + ]; + + if (cfg.requiredEnvVar && process.env[cfg.requiredEnvVar]) { + createArgs.push('--api-key-arn', process.env[cfg.requiredEnvVar]!); + } + + if (cfg.skipMemory) { + createArgs.push('--no-harness-memory'); + } + + const result = await runAgentCoreCLI(createArgs, testDir); + + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { projectPath: string }; + projectPath = json.projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, harnessName, cfg.modelProvider); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + it.skipIf(!canRun)( + 'deploys to AWS successfully', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr}, stdout: ${result.stdout})`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Deploy should report success').toBe(true); + }, + 1, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed harness', + async () => { + expect(projectPath, 'Project should have been created').toBeTruthy(); + + await retry( + async () => { + const result = await runAgentCoreCLI( + ['invoke', '--harness', harnessName, '--prompt', 'Say hello', '--json'], + projectPath + ); + + if (result.exitCode !== 0) { + console.log('Invoke stdout:', result.stdout); + console.log('Invoke stderr:', result.stderr); + } + + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success, 'Invoke should report success').toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'status shows the deployed harness', + async () => { + const statusResult = await spawnAndCollect('agentcore', ['status', '--json'], projectPath); + + expect(statusResult.exitCode, `Status failed: ${statusResult.stderr}`).toBe(0); + + const json = parseJsonOutput(statusResult.stdout) as { + success: boolean; + resources: { + resourceType: string; + name: string; + deploymentState: string; + identifier?: string; + }[]; + }; + expect(json.success).toBe(true); + + const harness = json.resources.find(r => r.resourceType === 'harness' && r.name === harnessName); + expect(harness, `Harness "${harnessName}" should appear in status`).toBeDefined(); + expect(harness!.deploymentState).toBe('deployed'); + expect(harness!.identifier, 'Deployed harness should have a harnessArn').toBeTruthy(); + }, + 120000 + ); + }); +} diff --git a/e2e-tests/harness-gemini.test.ts b/e2e-tests/harness-gemini.test.ts new file mode 100644 index 000000000..8fd024147 --- /dev/null +++ b/e2e-tests/harness-gemini.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'gemini', requiredEnvVar: 'GEMINI_API_KEY_ARN', skipMemory: true }); diff --git a/e2e-tests/harness-openai.test.ts b/e2e-tests/harness-openai.test.ts new file mode 100644 index 000000000..bdb9c3772 --- /dev/null +++ b/e2e-tests/harness-openai.test.ts @@ -0,0 +1,3 @@ +import { createHarnessE2ESuite } from './harness-e2e-helper.js'; + +createHarnessE2ESuite({ modelProvider: 'open_ai', requiredEnvVar: 'OPENAI_API_KEY_ARN', skipMemory: true }); diff --git a/e2e-tests/http-gateway-targets.test.ts b/e2e-tests/http-gateway-targets.test.ts new file mode 100644 index 000000000..c2bef22fb --- /dev/null +++ b/e2e-tests/http-gateway-targets.test.ts @@ -0,0 +1,228 @@ +import { parseJsonOutput, retry } from '../src/test-utils/index.js'; +import { + baseCanRun, + hasAws, + installCdkTarball, + runAgentCoreCLI, + teardownE2EProject, + writeAwsTargets, +} from './e2e-helper.js'; +import { randomUUID } from 'node:crypto'; +import { mkdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const canRun = baseCanRun && hasAws; + +describe.sequential('e2e: HTTP gateway with targets lifecycle', () => { + let testDir: string; + let projectPath: string; + const agentName = `E2eGwTgt${String(Date.now()).slice(-8)}`; + const gatewayName = 'e2e-target-gw'; + + beforeAll(async () => { + if (!canRun) return; + + testDir = join(tmpdir(), `agentcore-e2e-gw-targets-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const result = await runAgentCoreCLI( + [ + 'create', + '--name', + agentName, + '--language', + 'Python', + '--framework', + 'Strands', + '--model-provider', + 'Bedrock', + '--memory', + 'none', + '--json', + ], + testDir + ); + expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0); + projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 300000); + + afterAll(async () => { + if (projectPath && hasAws) { + await teardownE2EProject(projectPath, agentName, 'Bedrock'); + } + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600000); + + const run = (args: string[]) => runAgentCoreCLI(args, projectPath); + + it.skipIf(!canRun)( + 'adds runtime endpoints (prod, staging)', + async () => { + let result = await run([ + 'add', + 'runtime-endpoint', + '--runtime', + agentName, + '--endpoint', + 'prod', + '--version', + '1', + '--json', + ]); + expect(result.exitCode, `Add prod endpoint failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'runtime-endpoint', + '--runtime', + agentName, + '--endpoint', + 'staging', + '--version', + '1', + '--json', + ]); + expect(result.exitCode, `Add staging endpoint failed: ${result.stdout}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds HTTP gateway with name', + async () => { + const result = await run(['add', 'gateway', '--name', gatewayName, '--json']); + expect(result.exitCode, `Add gateway failed: ${result.stdout}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'adds gateway targets for prod and staging endpoints', + async () => { + let result = await run([ + 'add', + 'gateway-target', + '--name', + `${agentName}-prod`, + '--type', + 'mcp-server', + '--endpoint', + 'https://placeholder-prod.example.com', + '--gateway', + gatewayName, + '--json', + ]); + expect(result.exitCode, `Add prod target failed: ${result.stdout}`).toBe(0); + + result = await run([ + 'add', + 'gateway-target', + '--name', + `${agentName}-staging`, + '--type', + 'mcp-server', + '--endpoint', + 'https://placeholder-staging.example.com', + '--gateway', + gatewayName, + '--json', + ]); + expect(result.exitCode, `Add staging target failed: ${result.stdout}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'deploys project with gateway and targets', + async () => { + await retry( + async () => { + const result = await run(['deploy', '--yes', '--json']); + if (result.exitCode !== 0) { + console.log('Deploy stdout:', result.stdout); + console.log('Deploy stderr:', result.stderr); + } + expect(result.exitCode, `Deploy failed (stderr: ${result.stderr})`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 2, + 30000 + ); + }, + 600000 + ); + + it.skipIf(!canRun)( + 'status shows gateway deployed', + async () => { + await retry( + async () => { + const result = await run(['status', '--json']); + expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string; identifier?: string }[]; + }; + expect(json.success).toBe(true); + + // Agent should be deployed + const agent = json.resources.find(r => r.resourceType === 'agent' && r.name === agentName); + expect(agent, `Agent "${agentName}" should appear in status`).toBeDefined(); + expect(agent!.deploymentState).toBe('deployed'); + }, + 3, + 15000 + ); + }, + 120000 + ); + + it.skipIf(!canRun)( + 'invokes the deployed agent directly', + async () => { + await retry( + async () => { + const result = await run(['invoke', '--prompt', 'Say hello', '--runtime', agentName, '--json']); + expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 3, + 15000 + ); + }, + 180000 + ); + + it.skipIf(!canRun)( + 'removes gateway targets', + async () => { + let result = await run(['remove', 'gateway-target', '--name', `${agentName}-prod`, '--json']); + expect(result.exitCode, `Remove prod target failed: ${result.stderr}`).toBe(0); + + result = await run(['remove', 'gateway-target', '--name', `${agentName}-staging`, '--json']); + expect(result.exitCode, `Remove staging target failed: ${result.stderr}`).toBe(0); + }, + 60000 + ); + + it.skipIf(!canRun)( + 'removes gateway', + async () => { + const result = await run(['remove', 'gateway', '--name', gatewayName, '--json']); + expect(result.exitCode, `Remove gateway failed: ${result.stderr}`).toBe(0); + const json = parseJsonOutput(result.stdout) as { success: boolean }; + expect(json.success).toBe(true); + }, + 60000 + ); +}); diff --git a/e2e-tests/import-gateway.test.ts b/e2e-tests/import-gateway.test.ts index 2aea04f02..fd80ae967 100644 --- a/e2e-tests/import-gateway.test.ts +++ b/e2e-tests/import-gateway.test.ts @@ -43,6 +43,7 @@ describe.sequential('e2e: import gateway', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'setup_gateway.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -50,7 +51,7 @@ describe.sequential('e2e: import gateway', () => { ); } - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; gatewayArn = resources.gateway!.arn; @@ -80,6 +81,7 @@ describe.sequential('e2e: import gateway', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ diff --git a/e2e-tests/import-resources.test.ts b/e2e-tests/import-resources.test.ts index d51cbffac..72d9c253a 100644 --- a/e2e-tests/import-resources.test.ts +++ b/e2e-tests/import-resources.test.ts @@ -54,6 +54,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', script], fixtureDir, { AWS_REGION: region, DEFAULT_EVALUATOR_MODEL, + RESOURCE_SUFFIX: suffix, }); if (result.exitCode !== 0) { throw new Error( @@ -63,7 +64,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { } // 2. Read resource ARNs from bugbash-resources.json - const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resourcesPath = join(fixtureDir, `bugbash-resources-${suffix}.json`); const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; runtimeArn = resources['runtime-basic']!.arn; memoryArn = resources['memory-full']!.arn; @@ -102,6 +103,7 @@ describe.sequential('e2e: import runtime/memory/evaluator', () => { try { await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { AWS_REGION: region, + RESOURCE_SUFFIX: suffix, }); } catch { /* ignore — resources may already be deleted by CFN teardown */ diff --git a/esbuild.config.mjs b/esbuild.config.mjs index 91e557270..2cbd5b81f 100644 --- a/esbuild.config.mjs +++ b/esbuild.config.mjs @@ -51,7 +51,7 @@ await esbuild.build({ jsx: 'automatic', // Inject require shim for ESM compatibility with CommonJS dependencies banner: { - js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`, + js: `import { createRequire } from 'module'; import { fileURLToPath as __ef } from 'url'; import { dirname as __ed } from 'path'; const require = createRequire(import.meta.url); const __filename = __ef(import.meta.url); const __dirname = __ed(__filename);`, }, external: ['fsevents', '@aws-cdk/toolkit-lib'], plugins: [optionalDepsPlugin, textLoaderPlugin], diff --git a/integ-tests/add-remove-ab-test-target-based.test.ts b/integ-tests/add-remove-ab-test-target-based.test.ts new file mode 100644 index 000000000..8a77b1f06 --- /dev/null +++ b/integ-tests/add-remove-ab-test-target-based.test.ts @@ -0,0 +1,461 @@ +import { + type TestProject, + createTestProject, + parseJsonOutput, + readProjectConfig, + runCLI, +} from '../src/test-utils/index.js'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function runSuccess(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', true); + return json as Record; +} + +async function runFailure(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode).toBe(1); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', false); + expect(json).toHaveProperty('error'); + return json as Record; +} + +describe('integration: add and remove target-based ab-test', () => { + let project: TestProject; + const gatewayName = 'my-test-gw'; + + beforeAll(async () => { + project = await createTestProject({ + name: 'TargetABTest', + language: 'Python', + framework: 'Strands', + modelProvider: 'Bedrock', + memory: 'none', + }); + + // Add runtime endpoints (prod and staging) for the agent + await runSuccess( + ['add', 'runtime-endpoint', '--runtime', project.agentName, '--endpoint', 'prod', '--version', '1', '--json'], + project.projectPath + ); + await runSuccess( + ['add', 'runtime-endpoint', '--runtime', project.agentName, '--endpoint', 'staging', '--version', '1', '--json'], + project.projectPath + ); + + // Add an evaluator and two online eval configs (one per variant) + await runSuccess( + [ + 'add', + 'evaluator', + '--name', + 'TestEval', + '--level', + 'SESSION', + '--model', + 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + '--instructions', + 'Evaluate quality. Context: {context}', + '--json', + ], + project.projectPath + ); + await runSuccess( + [ + 'add', + 'online-eval', + '--name', + 'ControlEval', + '--runtime', + project.agentName, + '--evaluator', + 'TestEval', + '--sampling-rate', + '100', + '--endpoint', + 'prod', + '--json', + ], + project.projectPath + ); + await runSuccess( + [ + 'add', + 'online-eval', + '--name', + 'TreatmentEval', + '--runtime', + project.agentName, + '--evaluator', + 'TestEval', + '--sampling-rate', + '100', + '--endpoint', + 'staging', + '--json', + ], + project.projectPath + ); + }, 120000); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds target-based AB test with --control-endpoint and --treatment-endpoint', async () => { + const json = await runSuccess( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'TargetTest1', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '90', + '--treatment-weight', + '10', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.abTestName).toBe('TargetTest1'); + + // Verify agentcore.json has correct mode, targets, gateway auto-created + const spec = await readProjectConfig(project.projectPath); + const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'TargetTest1'); + expect(abTest).toBeDefined(); + expect(abTest!.mode).toBe('target-based'); + expect(abTest!.variants).toHaveLength(2); + expect(abTest!.variants[0]!.name).toBe('C'); + expect(abTest!.variants[0]!.weight).toBe(90); + expect(abTest!.variants[0]!.variantConfiguration.target).toBeDefined(); + expect(abTest!.variants[0]!.variantConfiguration.target!.targetName).toBe(`${project.agentName}-prod`); + expect(abTest!.variants[1]!.name).toBe('T1'); + expect(abTest!.variants[1]!.weight).toBe(10); + expect(abTest!.variants[1]!.variantConfiguration.target!.targetName).toBe(`${project.agentName}-staging`); + expect(abTest!.gatewayRef).toBe(`{{gateway:${gatewayName}}}`); + + // Verify gateway was auto-created with targets + const gw = spec.httpGateways?.find((g: { name: string }) => g.name === gatewayName); + expect(gw, 'HTTP gateway should have been auto-created').toBeDefined(); + expect(gw!.targets).toBeDefined(); + expect(gw!.targets!.length).toBeGreaterThanOrEqual(2); + + const controlTarget = gw!.targets!.find((t: { name: string }) => t.name === `${project.agentName}-prod`); + expect(controlTarget).toBeDefined(); + expect(controlTarget!.qualifier).toBe('prod'); + + const treatmentTarget = gw!.targets!.find((t: { name: string }) => t.name === `${project.agentName}-staging`); + expect(treatmentTarget).toBeDefined(); + expect(treatmentTarget!.qualifier).toBe('staging'); + + // Verify per-variant evaluation config + const evalConfig = abTest!.evaluationConfig; + expect('perVariantOnlineEvaluationConfig' in evalConfig).toBe(true); + if ('perVariantOnlineEvaluationConfig' in evalConfig) { + expect(evalConfig.perVariantOnlineEvaluationConfig).toHaveLength(2); + const controlEval = evalConfig.perVariantOnlineEvaluationConfig.find( + (p: { treatmentName: string }) => p.treatmentName === 'C' + ); + expect(controlEval?.onlineEvaluationConfigArn).toBe('ControlEval'); + const treatmentEval = evalConfig.perVariantOnlineEvaluationConfig.find( + (p: { treatmentName: string }) => p.treatmentName === 'T1' + ); + expect(treatmentEval?.onlineEvaluationConfigArn).toBe('TreatmentEval'); + } + }); + + it('adds target-based AB test with existing gateway', async () => { + // TargetTest1 already created the gateway — reuse it + const json = await runSuccess( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'TargetTest2', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.abTestName).toBe('TargetTest2'); + + const spec = await readProjectConfig(project.projectPath); + // Gateway should still exist (reused, not duplicated) + const gateways = spec.httpGateways?.filter((g: { name: string }) => g.name === gatewayName); + expect(gateways).toHaveLength(1); + }); + + it('rejects duplicate AB test name', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'TargetTest1', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('already exists'); + }); + + it('rejects weights that do not sum to 100', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'BadWeights', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '80', + '--treatment-weight', + '80', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toBeDefined(); + }); + + it('errors when --control-endpoint is missing in target-based mode', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'MissingControl', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--treatment-endpoint', + 'staging', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('--control-endpoint'); + }); + + it('errors when --runtime is missing in target-based mode', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'MissingRuntime', + '--gateway', + gatewayName, + '--control-endpoint', + 'prod', + '--treatment-endpoint', + 'staging', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('--runtime'); + }); + + it('errors when endpoint does not exist on runtime', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'BadEndpoint', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-endpoint', + 'nonexistent', + '--treatment-endpoint', + 'staging', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('nonexistent'); + }); + + it('deprecated --control-qualifier still works as alias for --control-endpoint', async () => { + const json = await runSuccess( + [ + 'add', + 'ab-test', + '--mode', + 'target-based', + '--name', + 'QualifierAlias', + '--runtime', + project.agentName, + '--gateway', + gatewayName, + '--control-qualifier', + 'prod', + '--treatment-qualifier', + 'staging', + '--control-weight', + '60', + '--treatment-weight', + '40', + '--control-online-eval', + 'ControlEval', + '--treatment-online-eval', + 'TreatmentEval', + '--json', + ], + project.projectPath + ); + + expect(json.abTestName).toBe('QualifierAlias'); + + const spec = await readProjectConfig(project.projectPath); + const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'QualifierAlias'); + expect(abTest).toBeDefined(); + expect(abTest!.mode).toBe('target-based'); + expect(abTest!.variants[0]!.variantConfiguration.target!.targetName).toBe(`${project.agentName}-prod`); + expect(abTest!.variants[1]!.variantConfiguration.target!.targetName).toBe(`${project.agentName}-staging`); + }); + + it('removes target-based AB test without --delete-gateway', async () => { + const json = await runSuccess(['remove', 'ab-test', '--name', 'TargetTest2', '--json'], project.projectPath); + expect(json.success).toBe(true); + + // Verify removal from agentcore.json + const spec = await readProjectConfig(project.projectPath); + const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'TargetTest2'); + expect(abTest).toBeUndefined(); + + // Gateway should still exist (other AB tests reference it) + const gw = spec.httpGateways?.find((g: { name: string }) => g.name === gatewayName); + expect(gw, 'Gateway should still exist when other AB tests reference it').toBeDefined(); + }); + + it('removes target-based AB test with --delete-gateway flag', async () => { + // First remove QualifierAlias so only TargetTest1 is left referencing the gateway + await runSuccess(['remove', 'ab-test', '--name', 'QualifierAlias', '--json'], project.projectPath); + + // Now remove TargetTest1 with --delete-gateway + const json = await runSuccess( + ['remove', 'ab-test', '--name', 'TargetTest1', '--delete-gateway', '--json'], + project.projectPath + ); + expect(json.success).toBe(true); + + // Verify gateway was also removed (no other AB tests reference it) + const spec = await readProjectConfig(project.projectPath); + const gw = spec.httpGateways?.find((g: { name: string }) => g.name === gatewayName); + expect(gw, 'Gateway should be removed with --delete-gateway when no other AB tests reference it').toBeUndefined(); + }); + + it('remove returns error for non-existent test', async () => { + const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath); + expect(json.error).toContain('not found'); + }); +}); diff --git a/integ-tests/add-remove-ab-test.test.ts b/integ-tests/add-remove-ab-test.test.ts new file mode 100644 index 000000000..551c86010 --- /dev/null +++ b/integ-tests/add-remove-ab-test.test.ts @@ -0,0 +1,170 @@ +import { + type TestProject, + createTestProject, + parseJsonOutput, + readProjectConfig, + runCLI, +} from '../src/test-utils/index.js'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function runSuccess(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', true); + return json as Record; +} + +async function runFailure(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode).toBe(1); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', false); + expect(json).toHaveProperty('error'); + return json as Record; +} + +describe('integration: add and remove ab-test', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ + language: 'Python', + framework: 'Strands', + modelProvider: 'Bedrock', + memory: 'none', + }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('requires --name for JSON mode', async () => { + const json = await runFailure(['add', 'ab-test', '--json'], project.projectPath); + expect(json.error).toContain('--name'); + }); + + it('requires --runtime when --name is provided', async () => { + const json = await runFailure(['add', 'ab-test', '--name', 'Test1', '--json'], project.projectPath); + expect(json.error).toContain('--runtime'); + }); + + it('adds ab-test with all required flags', async () => { + const json = await runSuccess( + [ + 'add', + 'ab-test', + '--name', + 'MyIntegTest', + '--runtime', + project.agentName, + '--control-bundle', + 'arn:bundle:control', + '--control-version', + 'v1', + '--treatment-bundle', + 'arn:bundle:treatment', + '--treatment-version', + 'v1', + '--control-weight', + '80', + '--treatment-weight', + '20', + '--online-eval', + 'arn:eval:config', + '--json', + ], + project.projectPath + ); + + expect(json.abTestName).toBe('MyIntegTest'); + + // Verify it's in agentcore.json with correct structure + const spec = await readProjectConfig(project.projectPath); + const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest'); + expect(abTest).toBeDefined(); + expect(abTest!.variants).toHaveLength(2); + expect(abTest!.variants[0]!.name).toBe('C'); + expect(abTest!.variants[0]!.weight).toBe(80); + expect(abTest!.variants[1]!.name).toBe('T1'); + expect(abTest!.variants[1]!.weight).toBe(20); + }); + + it('rejects duplicate AB test name', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--name', + 'MyIntegTest', + '--runtime', + project.agentName, + '--control-bundle', + 'arn:cb', + '--control-version', + 'v1', + '--treatment-bundle', + 'arn:tb', + '--treatment-version', + 'v1', + '--control-weight', + '50', + '--treatment-weight', + '50', + '--online-eval', + 'arn:eval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('already exists'); + }); + + it('rejects weights that do not sum to 100', async () => { + const json = await runFailure( + [ + 'add', + 'ab-test', + '--name', + 'BadWeights', + '--runtime', + project.agentName, + '--control-bundle', + 'arn:cb', + '--control-version', + 'v1', + '--treatment-bundle', + 'arn:tb', + '--treatment-version', + 'v1', + '--control-weight', + '80', + '--treatment-weight', + '80', + '--online-eval', + 'arn:eval', + '--json', + ], + project.projectPath + ); + + expect(json.error).toBeDefined(); + }); + + it('removes ab-test', async () => { + const json = await runSuccess(['remove', 'ab-test', '--name', 'MyIntegTest', '--json'], project.projectPath); + expect(json.success).toBe(true); + + // Verify removal from agentcore.json + const spec = await readProjectConfig(project.projectPath); + const abTest = spec.abTests?.find((t: { name: string }) => t.name === 'MyIntegTest'); + expect(abTest).toBeUndefined(); + }); + + it('remove returns error for non-existent test', async () => { + const json = await runFailure(['remove', 'ab-test', '--name', 'DoesNotExist', '--json'], project.projectPath); + expect(json.error).toContain('not found'); + }); +}); diff --git a/integ-tests/add-remove-config-bundle.test.ts b/integ-tests/add-remove-config-bundle.test.ts new file mode 100644 index 000000000..bd53e7f31 --- /dev/null +++ b/integ-tests/add-remove-config-bundle.test.ts @@ -0,0 +1,312 @@ +import { + type TestProject, + createTestProject, + parseJsonOutput, + readProjectConfig, + runCLI, + runFailure, + runSuccess, +} from '../src/test-utils/index.js'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +describe('integration: add and remove config-bundle', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + // ── Add lifecycle ───────────────────────────────────────────────────── + + describe('add config-bundle', () => { + it('adds a config bundle with inline --components', async () => { + const components = JSON.stringify({ + 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-abc': { + configuration: { systemPrompt: 'You are a helpful assistant.' }, + }, + }); + + const json = await runSuccess( + ['add', 'config-bundle', '--name', 'InlineBundle', '--components', components, '--json'], + project.projectPath + ); + + expect(json.bundleName).toBe('InlineBundle'); + + const config = await readProjectConfig(project.projectPath); + const bundle = config.configBundles!.find(b => b.name === 'InlineBundle'); + expect(bundle).toBeDefined(); + expect(bundle!.type).toBe('ConfigurationBundle'); + expect(bundle!.branchName).toBe('mainline'); + expect(Object.keys(bundle!.components)).toHaveLength(1); + }); + + it('adds a config bundle with --components-file', async () => { + const componentsData = { + 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-def': { + configuration: { temperature: 0.7, maxTokens: 1024 }, + }, + 'arn:aws:bedrock-agentcore:us-east-1:123456789012:gateway/gw-xyz': { + configuration: { rateLimit: 100 }, + }, + }; + + const filePath = join(project.projectPath, 'test-components.json'); + await writeFile(filePath, JSON.stringify(componentsData)); + + const json = await runSuccess( + ['add', 'config-bundle', '--name', 'FileBundle', '--components-file', filePath, '--json'], + project.projectPath + ); + + expect(json.bundleName).toBe('FileBundle'); + + const config = await readProjectConfig(project.projectPath); + const bundle = config.configBundles!.find(b => b.name === 'FileBundle'); + expect(bundle).toBeDefined(); + expect(Object.keys(bundle!.components)).toHaveLength(2); + }); + + it('adds a config bundle with optional description, branch, and commit message', async () => { + const components = JSON.stringify({ + '{{runtime:MyAgent}}': { + configuration: { systemPrompt: 'Placeholder-based bundle' }, + }, + }); + + const json = await runSuccess( + [ + 'add', + 'config-bundle', + '--name', + 'FullOptsBundle', + '--description', + 'A bundle with all optional fields', + '--components', + components, + '--branch', + 'feature-branch', + '--commit-message', + 'initial config', + '--json', + ], + project.projectPath + ); + + expect(json.bundleName).toBe('FullOptsBundle'); + + const config = await readProjectConfig(project.projectPath); + const bundle = config.configBundles!.find(b => b.name === 'FullOptsBundle'); + expect(bundle).toBeDefined(); + expect(bundle!.description).toBe('A bundle with all optional fields'); + expect(bundle!.branchName).toBe('feature-branch'); + expect(bundle!.commitMessage).toBe('initial config'); + }); + + it('adds a config bundle with placeholder component keys', async () => { + const components = JSON.stringify({ + '{{runtime:AgentA}}': { + configuration: { systemPrompt: 'Runtime placeholder' }, + }, + '{{gateway:GatewayB}}': { + configuration: { rateLimitPerSecond: 50 }, + }, + }); + + const json = await runSuccess( + ['add', 'config-bundle', '--name', 'PlaceholderBundle', '--components', components, '--json'], + project.projectPath + ); + + expect(json.bundleName).toBe('PlaceholderBundle'); + + const config = await readProjectConfig(project.projectPath); + const bundle = config.configBundles!.find(b => b.name === 'PlaceholderBundle'); + expect(bundle).toBeDefined(); + const keys = Object.keys(bundle!.components); + expect(keys).toContain('{{runtime:AgentA}}'); + expect(keys).toContain('{{gateway:GatewayB}}'); + }); + }); + + // ── Validation / error cases ────────────────────────────────────────── + + describe('validation errors', () => { + it('rejects duplicate config bundle name', async () => { + const components = JSON.stringify({ + 'arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-dup': { + configuration: { foo: 'bar' }, + }, + }); + + const json = await runFailure( + ['add', 'config-bundle', '--name', 'InlineBundle', '--components', components, '--json'], + project.projectPath + ); + + expect(json.error).toContain('already exists'); + }); + + it('requires --name in non-interactive (JSON) mode', async () => { + const result = await runCLI( + ['add', 'config-bundle', '--components', '{"arn:test": {"configuration": {}}}', '--json'], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('--name'); + }); + + it('requires --components or --components-file when --name is provided', async () => { + const json = await runFailure(['add', 'config-bundle', '--name', 'NoComponents', '--json'], project.projectPath); + + expect(json.error).toContain('--components'); + }); + + it('rejects invalid JSON in --components', async () => { + const result = await runCLI( + ['add', 'config-bundle', '--name', 'BadJson', '--components', '{not valid json}', '--json'], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + }); + + it('rejects --components-file with non-existent file', async () => { + const result = await runCLI( + [ + 'add', + 'config-bundle', + '--name', + 'MissingFile', + '--components-file', + '/tmp/does-not-exist-xyz.json', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + }); + + it('rejects bundle name with invalid characters', async () => { + const components = JSON.stringify({ + 'arn:test': { configuration: {} }, + }); + + const json = await runFailure( + ['add', 'config-bundle', '--name', 'invalid-name!', '--components', components, '--json'], + project.projectPath + ); + + expect(json.error).toBeDefined(); + }); + + it('rejects bundle name starting with a number', async () => { + const components = JSON.stringify({ + 'arn:test': { configuration: {} }, + }); + + const json = await runFailure( + ['add', 'config-bundle', '--name', '1BadName', '--components', components, '--json'], + project.projectPath + ); + + expect(json.error).toBeDefined(); + }); + }); + + // ── Remove lifecycle ────────────────────────────────────────────────── + + describe('remove config-bundle', () => { + it('removes an existing config bundle', async () => { + const json = await runSuccess( + ['remove', 'config-bundle', '--name', 'InlineBundle', '--json'], + project.projectPath + ); + + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const bundle = config.configBundles!.find(b => b.name === 'InlineBundle'); + expect(bundle).toBeUndefined(); + }); + + it('returns error for non-existent bundle', async () => { + const json = await runFailure( + ['remove', 'config-bundle', '--name', 'DoesNotExist', '--json'], + project.projectPath + ); + + expect(json.error).toContain('not found'); + }); + + it('removes all remaining config bundles one by one', async () => { + const configBefore = await readProjectConfig(project.projectPath); + const remaining = configBefore.configBundles!.map(b => b.name); + + for (const name of remaining) { + await runSuccess(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath); + } + + const configAfter = await readProjectConfig(project.projectPath); + expect(configAfter.configBundles!).toHaveLength(0); + }); + }); + + // ── Multiple bundles coexistence ────────────────────────────────────── + + describe('multiple bundles coexistence', () => { + const bundleNames = ['BundleAlpha', 'BundleBeta', 'BundleGamma']; + + it('can add multiple config bundles to the same project', async () => { + for (const name of bundleNames) { + const components = JSON.stringify({ + [`arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/${name}`]: { + configuration: { bundleId: name }, + }, + }); + + await runSuccess( + ['add', 'config-bundle', '--name', name, '--components', components, '--json'], + project.projectPath + ); + } + + const config = await readProjectConfig(project.projectPath); + expect(config.configBundles!).toHaveLength(bundleNames.length); + + for (const name of bundleNames) { + expect(config.configBundles!.find(b => b.name === name)).toBeDefined(); + } + }); + + it('removing one bundle does not affect others', async () => { + await runSuccess(['remove', 'config-bundle', '--name', 'BundleBeta', '--json'], project.projectPath); + + const config = await readProjectConfig(project.projectPath); + expect(config.configBundles!).toHaveLength(2); + expect(config.configBundles!.find(b => b.name === 'BundleAlpha')).toBeDefined(); + expect(config.configBundles!.find(b => b.name === 'BundleGamma')).toBeDefined(); + expect(config.configBundles!.find(b => b.name === 'BundleBeta')).toBeUndefined(); + }); + + afterAll(async () => { + for (const name of bundleNames) { + try { + await runCLI(['remove', 'config-bundle', '--name', name, '--json'], project.projectPath); + } catch { + // already removed + } + } + }); + }); +}); diff --git a/integ-tests/add-remove-harness.test.ts b/integ-tests/add-remove-harness.test.ts new file mode 100644 index 000000000..2f69270db --- /dev/null +++ b/integ-tests/add-remove-harness.test.ts @@ -0,0 +1,204 @@ +import { createTestProject, exists, readProjectConfig, runCLI } from '../src/test-utils/index.js'; +import type { TestProject } from '../src/test-utils/index.js'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function readHarnessSpec(projectPath: string, harnessName: string) { + return JSON.parse(await readFile(join(projectPath, `app/${harnessName}/harness.json`), 'utf-8')); +} + +describe('integration: harness add/remove lifecycle', () => { + let project: TestProject; + const harnessName = 'TestHarness'; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds a harness with defaults', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness, `Harness "${harnessName}" should be in agentcore.json`).toBeTruthy(); + expect(harness!.path).toBe(`app/${harnessName}`); + }); + + it('creates harness.json with correct model config', async () => { + const spec = await readHarnessSpec(project.projectPath, harnessName); + expect(spec.model).toBeDefined(); + expect(spec.model.provider).toBe('bedrock'); + expect(spec.model.modelId).toBeTruthy(); + }); + + it('creates system-prompt.md', async () => { + const promptPath = join(project.projectPath, `app/${harnessName}/system-prompt.md`); + expect(await exists(promptPath), 'system-prompt.md should exist').toBe(true); + }); + + it('auto-creates memory resource', async () => { + const config = await readProjectConfig(project.projectPath); + const memories = config.memories ?? []; + expect(memories.length, 'Should have auto-created memory').toBeGreaterThan(0); + }); + + it('rejects duplicate harness name', async () => { + const result = await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('removes the harness', async () => { + const result = await runCLI(['remove', 'harness', '--name', harnessName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + const config = await readProjectConfig(project.projectPath); + const found = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(found, `Harness "${harnessName}" should be removed`).toBeFalsy(); + }); +}); + +describe('integration: harness configuration options', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds harness with truncation strategy', async () => { + const name = 'TruncHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--truncation-strategy', 'sliding_window', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.truncation?.strategy).toBe('sliding_window'); + }); + + it('adds harness with lifecycle config', async () => { + const name = 'LifecycleHarness'; + const result = await runCLI( + ['add', 'harness', '--name', name, '--idle-timeout', '300', '--max-lifetime', '3600', '--json'], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.lifecycleConfig?.idleRuntimeSessionTimeout).toBe(300); + expect(spec.lifecycleConfig?.maxLifetime).toBe(3600); + }); + + it('adds harness without memory when --no-memory is set', async () => { + const name = 'NoMemHarness'; + const configBefore = await readProjectConfig(project.projectPath); + const memoriesBefore = (configBefore.memories ?? []).length; + + const result = await runCLI(['add', 'harness', '--name', name, '--no-memory', '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const configAfter = await readProjectConfig(project.projectPath); + const memoriesAfter = (configAfter.memories ?? []).length; + expect(memoriesAfter).toBe(memoriesBefore); + }); + + it('adds harness with non-bedrock model provider', async () => { + const name = 'OpenAIHarness'; + const result = await runCLI( + [ + 'add', + 'harness', + '--name', + name, + '--model-provider', + 'open_ai', + '--model-id', + 'gpt-5', + '--api-key-arn', + 'arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const spec = await readHarnessSpec(project.projectPath, name); + expect(spec.model.provider).toBe('open_ai'); + expect(spec.model.modelId).toBe('gpt-5'); + expect(spec.model.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123456789012:secret:openai-key'); + }); +}); + +describe('integration: harness validation errors', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ noAgent: true }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('rejects invalid harness name with special characters', async () => { + const result = await runCLI(['add', 'harness', '--name', 'bad-name!', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects harness name starting with a number', async () => { + const result = await runCLI(['add', 'harness', '--name', '1BadName', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); + + it('rejects add harness without --name when --json is passed', async () => { + const result = await runCLI(['add', 'harness', '--json'], project.projectPath); + expect(result.exitCode).not.toBe(0); + }); +}); + +describe('integration: create project with harness', () => { + let project: TestProject; + const harnessName = 'CreateHarness'; + + beforeAll(async () => { + project = await createTestProject({ name: harnessName, noAgent: true }); + await runCLI(['add', 'harness', '--name', harnessName, '--json'], project.projectPath); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + it('has correct project scaffolding', async () => { + expect(await exists(join(project.projectPath, 'agentcore/agentcore.json'))).toBe(true); + expect(await exists(join(project.projectPath, 'agentcore/cdk'))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/harness.json`))).toBe(true); + expect(await exists(join(project.projectPath, `app/${harnessName}/system-prompt.md`))).toBe(true); + }); + + it('has harness registered in project config', async () => { + const config = await readProjectConfig(project.projectPath); + const harness = config.harnesses?.find((h: { name: string }) => h.name === harnessName); + expect(harness).toBeTruthy(); + }); +}); diff --git a/integ-tests/add-remove-online-eval-endpoint.test.ts b/integ-tests/add-remove-online-eval-endpoint.test.ts new file mode 100644 index 000000000..cb2a614c8 --- /dev/null +++ b/integ-tests/add-remove-online-eval-endpoint.test.ts @@ -0,0 +1,199 @@ +import { + type TestProject, + createTestProject, + parseJsonOutput, + readProjectConfig, + runCLI, +} from '../src/test-utils/index.js'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +async function runSuccess(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', true); + return json as Record; +} + +async function runFailure(args: string[], cwd: string) { + const result = await runCLI(args, cwd); + expect(result.exitCode).toBe(1); + const json: unknown = parseJsonOutput(result.stdout); + expect(json).toHaveProperty('success', false); + expect(json).toHaveProperty('error'); + return json as Record; +} + +describe('integration: add and remove online-eval with endpoint', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ + name: 'OnlineEvalEP', + language: 'Python', + framework: 'Strands', + modelProvider: 'Bedrock', + memory: 'none', + }); + + // Add runtime endpoints (prod and staging) for the agent + await runSuccess( + ['add', 'runtime-endpoint', '--runtime', project.agentName, '--endpoint', 'prod', '--version', '1', '--json'], + project.projectPath + ); + await runSuccess( + ['add', 'runtime-endpoint', '--runtime', project.agentName, '--endpoint', 'staging', '--version', '1', '--json'], + project.projectPath + ); + + // Add an evaluator to reference in online eval configs + await runSuccess( + [ + 'add', + 'evaluator', + '--name', + 'QualityEval', + '--level', + 'SESSION', + '--model', + 'us.anthropic.claude-sonnet-4-5-20250929-v1:0', + '--instructions', + 'Evaluate quality. Context: {context}', + '--json', + ], + project.projectPath + ); + }, 120000); + + afterAll(async () => { + await project.cleanup(); + }); + + it('adds online eval with --endpoint prod', async () => { + const json = await runSuccess( + [ + 'add', + 'online-eval', + '--name', + 'ProdEval', + '--runtime', + project.agentName, + '--evaluator', + 'QualityEval', + '--sampling-rate', + '100', + '--endpoint', + 'prod', + '--json', + ], + project.projectPath + ); + + expect(json.configName).toBe('ProdEval'); + + // Verify agentcore.json has endpoint field + const spec = await readProjectConfig(project.projectPath); + const evalConfig = spec.onlineEvalConfigs?.find((c: { name: string }) => c.name === 'ProdEval'); + expect(evalConfig).toBeDefined(); + expect(evalConfig!.endpoint).toBe('prod'); + expect(evalConfig!.agent).toBe(project.agentName); + expect(evalConfig!.evaluators).toContain('QualityEval'); + expect(evalConfig!.samplingRate).toBe(100); + }); + + it('adds online eval with --endpoint staging', async () => { + const json = await runSuccess( + [ + 'add', + 'online-eval', + '--name', + 'StagingEval', + '--runtime', + project.agentName, + '--evaluator', + 'QualityEval', + '--sampling-rate', + '50', + '--endpoint', + 'staging', + '--json', + ], + project.projectPath + ); + + expect(json.configName).toBe('StagingEval'); + + const spec = await readProjectConfig(project.projectPath); + const evalConfig = spec.onlineEvalConfigs?.find((c: { name: string }) => c.name === 'StagingEval'); + expect(evalConfig).toBeDefined(); + expect(evalConfig!.endpoint).toBe('staging'); + }); + + it('adds online eval without --endpoint (no endpoint field in config)', async () => { + const json = await runSuccess( + [ + 'add', + 'online-eval', + '--name', + 'NoEndpointEval', + '--runtime', + project.agentName, + '--evaluator', + 'QualityEval', + '--sampling-rate', + '100', + '--json', + ], + project.projectPath + ); + + expect(json.configName).toBe('NoEndpointEval'); + + const spec = await readProjectConfig(project.projectPath); + const evalConfig = spec.onlineEvalConfigs?.find((c: { name: string }) => c.name === 'NoEndpointEval'); + expect(evalConfig).toBeDefined(); + expect(evalConfig!.endpoint).toBeUndefined(); + }); + + it('errors when endpoint does not exist on runtime', async () => { + const json = await runFailure( + [ + 'add', + 'online-eval', + '--name', + 'BadEndpointEval', + '--runtime', + project.agentName, + '--evaluator', + 'QualityEval', + '--sampling-rate', + '100', + '--endpoint', + 'nonexistent', + '--json', + ], + project.projectPath + ); + + expect(json.error).toContain('nonexistent'); + }); + + it('removes online eval config', async () => { + const json = await runSuccess(['remove', 'online-eval', '--name', 'ProdEval', '--json'], project.projectPath); + expect(json.success).toBe(true); + + // Verify removal from agentcore.json + const spec = await readProjectConfig(project.projectPath); + const evalConfig = spec.onlineEvalConfigs?.find((c: { name: string }) => c.name === 'ProdEval'); + expect(evalConfig).toBeUndefined(); + + // Other eval configs should remain + const stagingEval = spec.onlineEvalConfigs?.find((c: { name: string }) => c.name === 'StagingEval'); + expect(stagingEval).toBeDefined(); + }); + + it('remove returns error for non-existent online eval', async () => { + const json = await runFailure(['remove', 'online-eval', '--name', 'DoesNotExist', '--json'], project.projectPath); + expect(json.error).toContain('not found'); + }); +}); diff --git a/integ-tests/add-remove-resources.test.ts b/integ-tests/add-remove-resources.test.ts index 57dd48483..a89c761dd 100644 --- a/integ-tests/add-remove-resources.test.ts +++ b/integ-tests/add-remove-resources.test.ts @@ -1,7 +1,10 @@ import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import type { TestProject } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +const telemetry = createTelemetryHelper(); + describe('integration: add and remove resources', () => { let project: TestProject; @@ -16,13 +19,16 @@ describe('integration: add and remove resources', () => { afterAll(async () => { await project.cleanup(); + telemetry.destroy(); }); describe('memory lifecycle', () => { const memoryName = `IntegMem${Date.now().toString().slice(-6)}`; it('adds a memory resource', async () => { - const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath); + const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, { + env: telemetry.env, + }); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); const json = JSON.parse(result.stdout); @@ -34,13 +40,17 @@ describe('integration: add and remove resources', () => { expect(memories, 'memories should exist').toBeDefined(); const found = memories!.some((m: Record) => m.name === memoryName); expect(found, `Memory "${memoryName}" should be in config`).toBe(true); + + // Verify telemetry + telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success' }); }); it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => { const episodicMemName = `EpiMem${Date.now().toString().slice(-6)}`; const result = await runCLI( ['add', 'memory', '--name', episodicMemName, '--strategies', 'EPISODIC', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -61,6 +71,14 @@ describe('integration: add and remove resources', () => { expect(episodic!.reflectionNamespaces, 'Should have reflectionNamespaces').toBeDefined(); expect(episodic!.reflectionNamespaces!.length).toBeGreaterThan(0); + // Verify telemetry + telemetry.assertMetricEmitted({ + command: 'add.memory', + exit_reason: 'success', + strategy_count: '1', + strategy_episodic: 'true', + }); + // Clean up await runCLI(['remove', 'memory', '--name', episodicMemName, '--json'], project.projectPath); }); @@ -86,7 +104,8 @@ describe('integration: add and remove resources', () => { it('adds a credential resource', async () => { const result = await runCLI( ['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'], - project.projectPath + project.projectPath, + { env: telemetry.env } ); expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); @@ -99,6 +118,13 @@ describe('integration: add and remove resources', () => { expect(credentials, 'credentials should exist').toBeDefined(); const found = credentials!.some((c: Record) => c.name === credentialName); expect(found, `Credential "${credentialName}" should be in config`).toBe(true); + + // Verify telemetry + telemetry.assertMetricEmitted({ + command: 'add.credential', + exit_reason: 'success', + credential_type: 'api-key', + }); }); it('removes the credential resource', async () => { @@ -115,4 +141,30 @@ describe('integration: add and remove resources', () => { expect(found, `Credential "${credentialName}" should be removed from config`).toBe(false); }); }); + + describe('policy-engine', () => { + const engineName = `TestEngine${Date.now().toString().slice(-6)}`; + + it('adds a policy engine resource', async () => { + const result = await runCLI(['add', 'policy-engine', '--name', engineName, '--json'], project.projectPath, { + env: telemetry.env, + }); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + + telemetry.assertMetricEmitted({ + command: 'add.policy-engine', + exit_reason: 'success', + attach_gateway_count: '0', + }); + }); + + it('removes the policy engine resource', async () => { + const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + }); + }); }); diff --git a/integ-tests/create-edge-cases.test.ts b/integ-tests/create-edge-cases.test.ts index 5d7a1b4e5..d1bbcf056 100644 --- a/integ-tests/create-edge-cases.test.ts +++ b/integ-tests/create-edge-cases.test.ts @@ -131,7 +131,10 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', it('--dry-run shows what would be created without writing files', async () => { const name = `DryRun${Date.now().toString().slice(-6)}`; - const result = await runCLI(['create', '--name', name, '--defaults', '--dry-run', '--json'], testDir); + const result = await runCLI( + ['create', '--name', name, '--framework', 'Strands', '--defaults', '--dry-run', '--json'], + testDir + ); expect(result.exitCode).toBe(0); const json = JSON.parse(result.stdout); diff --git a/integ-tests/create-frameworks.test.ts b/integ-tests/create-frameworks.test.ts index dee93cc1e..82bbc0871 100644 --- a/integ-tests/create-frameworks.test.ts +++ b/integ-tests/create-frameworks.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable security/detect-non-literal-fs-filename */ import { exists, prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm } from 'node:fs/promises'; diff --git a/integ-tests/create-memory.test.ts b/integ-tests/create-memory.test.ts index 35cd4436d..ac80f1ba4 100644 --- a/integ-tests/create-memory.test.ts +++ b/integ-tests/create-memory.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable security/detect-non-literal-fs-filename */ import { prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import { randomUUID } from 'node:crypto'; import { mkdir, rm } from 'node:fs/promises'; diff --git a/integ-tests/create-no-agent.test.ts b/integ-tests/create-no-agent.test.ts index 4bcca2690..bcdf80eaa 100644 --- a/integ-tests/create-no-agent.test.ts +++ b/integ-tests/create-no-agent.test.ts @@ -32,7 +32,7 @@ describe('integration: create without agent', () => { it.skipIf(!hasNpm || !hasGit)('creates project with real npm install and git init', async () => { const name = `NoAgent${Date.now().toString().slice(-6)}`; - const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, false); + const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, { skipInstall: false }); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/create-protocols.test.ts b/integ-tests/create-protocols.test.ts index 30b707f8c..440050fdb 100644 --- a/integ-tests/create-protocols.test.ts +++ b/integ-tests/create-protocols.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable security/detect-non-literal-fs-filename */ import { exists, prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js'; import { randomUUID } from 'node:crypto'; import { mkdir, readFile, rm } from 'node:fs/promises'; diff --git a/integ-tests/create-with-agent.test.ts b/integ-tests/create-with-agent.test.ts index 7fb20bdbf..69f0594b8 100644 --- a/integ-tests/create-with-agent.test.ts +++ b/integ-tests/create-with-agent.test.ts @@ -49,7 +49,7 @@ describe('integration: create with Python agent', () => { '--json', ], testDir, - false + { skipInstall: false } ); expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0); diff --git a/integ-tests/dev-server.test.ts b/integ-tests/dev-server.test.ts index 5f60976e7..4b07b7284 100644 --- a/integ-tests/dev-server.test.ts +++ b/integ-tests/dev-server.test.ts @@ -60,7 +60,7 @@ describe('integration: dev server', () => { '--json', ], testDir, - false + { skipInstall: false } ); if (result.exitCode === 0) { diff --git a/integ-tests/help.test.ts b/integ-tests/help.test.ts index 052605c7a..7e2176e2f 100644 --- a/integ-tests/help.test.ts +++ b/integ-tests/help.test.ts @@ -1,10 +1,9 @@ import { spawnAndCollect } from '../src/test-utils/cli-runner.js'; import { runCLI } from '../src/test-utils/index.js'; +import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js'; import { readdirSync } from 'node:fs'; -import { mkdir, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it } from 'vitest'; const COMMANDS = [ 'create', @@ -45,52 +44,46 @@ describe('CLI help', () => { }); describe('help modes telemetry', () => { - let testConfigDir: string; + const telemetry = createTelemetryHelper(); const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); - beforeAll(async () => { - testConfigDir = join(tmpdir(), `agentcore-help-telemetry-${Date.now()}`); - await mkdir(testConfigDir, { recursive: true }); - }); - afterAll(() => rm(testConfigDir, { recursive: true, force: true })); + afterAll(() => telemetry.destroy()); function run(args: string[], extraEnv: Record = {}) { - return spawnAndCollect('node', [cliPath, ...args], tmpdir(), { + return spawnAndCollect('node', [cliPath, ...args], process.cwd(), { AGENTCORE_SKIP_INSTALL: '1', - AGENTCORE_CONFIG_DIR: testConfigDir, + ...telemetry.env, ...extraEnv, }); } it('writes JSONL audit file when audit is enabled via env var', async () => { - const result = await run(['help', 'modes'], { AGENTCORE_TELEMETRY_AUDIT: '1' }); + const result = await run(['help', 'modes']); expect(result.exitCode).toBe(0); - const telemetryDir = join(testConfigDir, 'telemetry'); - const files = readdirSync(telemetryDir).filter(f => f.startsWith('help-')); - expect(files).toHaveLength(1); - - const content = await readFile(join(telemetryDir, files[0]!), 'utf-8'); - const entry = JSON.parse(content.trim()); - expect(entry.attrs).toMatchObject({ - 'service.name': 'agentcore-cli', - 'agentcore-cli.mode': 'cli', + const entries = telemetry.readEntries(); + expect(entries).toHaveLength(1); + telemetry.assertMetricEmitted({ command_group: 'help', command: 'help.modes', exit_reason: 'success', }); - expect(entry.attrs['agentcore-cli.session_id']).toBeDefined(); - expect(entry.attrs['os.type']).toBeDefined(); - expect(entry.value).toBeGreaterThanOrEqual(0); + expect(entries[0]!.attrs['agentcore-cli.session_id']).toBeDefined(); + expect(entries[0]!.attrs['os.type']).toBeDefined(); + expect(entries[0]!.value).toBeGreaterThanOrEqual(0); }); it('does not write audit file when audit is not enabled', async () => { - const telemetryDir = join(testConfigDir, 'telemetry'); - await rm(telemetryDir, { recursive: true, force: true }); + telemetry.clearEntries(); - const result = await run(['help', 'modes']); + const noAuditCliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs'); + const result = await spawnAndCollect('node', [noAuditCliPath, 'help', 'modes'], process.cwd(), { + AGENTCORE_SKIP_INSTALL: '1', + AGENTCORE_CONFIG_DIR: telemetry.dir, + }); expect(result.exitCode).toBe(0); + const telemetryDir = join(telemetry.dir, 'telemetry'); try { const files = readdirSync(telemetryDir); expect(files).toHaveLength(0); diff --git a/integ-tests/json-output.test.ts b/integ-tests/json-output.test.ts index e4ef5835f..c54033026 100644 --- a/integ-tests/json-output.test.ts +++ b/integ-tests/json-output.test.ts @@ -41,8 +41,8 @@ describe('JSON output structure', () => { }); it('missing required options returns error JSON', async () => { - // Missing --language, --framework, etc without --no-agent - const result = await runCLI(['create', '--name', 'ValidName', '--json'], testDir); + // Missing --language, --model-provider, etc. on agent path + const result = await runCLI(['create', '--name', 'ValidName', '--framework', 'Strands', '--json'], testDir); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); diff --git a/integ-tests/recommendation.test.ts b/integ-tests/recommendation.test.ts new file mode 100644 index 000000000..dc3037a3e --- /dev/null +++ b/integ-tests/recommendation.test.ts @@ -0,0 +1,290 @@ +import { type TestProject, createTestProject, parseJsonOutput, runCLI } from '../src/test-utils/index.js'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +describe('integration: run recommendation CLI validation', () => { + let project: TestProject; + + beforeAll(async () => { + project = await createTestProject({ + language: 'Python', + framework: 'Strands', + modelProvider: 'Bedrock', + memory: 'none', + }); + }); + + afterAll(async () => { + await project.cleanup(); + }); + + describe('required flags', () => { + it('requires --runtime', async () => { + const result = await runCLI( + ['run', 'recommendation', '--evaluator', 'Builtin.Faithfulness', '--inline', 'test prompt', '--json'], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('--runtime'); + }); + + it('requires --evaluator for system-prompt type', async () => { + const result = await runCLI( + ['run', 'recommendation', '--runtime', project.agentName, '--inline', 'test prompt', '--json'], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('--evaluator'); + }); + + it('rejects invalid --type', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--type', + 'invalid-type', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'test prompt', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('--type'); + }); + }); + + describe('system-prompt recommendation input validation', () => { + it('fails when agent not deployed (inline input)', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant.', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('deployed'); + }); + + it('fails when agent not deployed (file input)', async () => { + const promptFile = join(project.projectPath, 'system-prompt.txt'); + await writeFile(promptFile, 'You are a helpful assistant for testing.'); + + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--prompt-file', + promptFile, + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('deployed'); + }); + + it('fails with non-existent prompt file', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--prompt-file', + '/tmp/nonexistent-prompt-file-xyz.txt', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + }); + }); + + describe('tool-description recommendation input validation', () => { + it('fails when agent not deployed (tool-description type with --tools)', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--type', + 'tool-description', + '--runtime', + project.agentName, + '--tools', + 'search:Searches the web for information', + '--tools', + 'calculator:Performs math calculations', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + expect(json.error).toContain('deployed'); + }); + }); + + describe('config bundle source validation', () => { + it('fails when bundle not found in deployed state', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--bundle-name', + 'NonExistentBundle', + '--bundle-version', + 'v1', + '--system-prompt-json-path', + 'systemPrompt', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.success).toBe(false); + // Fails at agent resolution (not deployed) before bundle resolution + expect(json.error).toContain('deployed'); + }); + }); + + describe('spans file validation', () => { + it('fails when spans file does not exist', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant.', + '--spans-file', + '/tmp/nonexistent-spans-xyz.json', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + }); + + it('fails when spans file contains invalid JSON', async () => { + const spansFile = join(project.projectPath, 'bad-spans.json'); + await writeFile(spansFile, 'not valid json'); + + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant.', + '--spans-file', + spansFile, + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + }); + }); + + describe('lookback and session options', () => { + it('accepts --lookback flag (fails at deploy check, not parsing)', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant.', + '--lookback', + '14', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.error).toContain('deployed'); + }); + + it('accepts --session-id flag (fails at deploy check, not parsing)', async () => { + const result = await runCLI( + [ + 'run', + 'recommendation', + '--runtime', + project.agentName, + '--evaluator', + 'Builtin.Faithfulness', + '--inline', + 'You are a helpful assistant.', + '--session-id', + 'sess-001', + 'sess-002', + '--json', + ], + project.projectPath + ); + + expect(result.exitCode).toBe(1); + const json = parseJsonOutput(result.stdout) as Record; + expect(json.error).toContain('deployed'); + }); + }); +}); diff --git a/integ-tests/tui/add-gateway-jwt.test.ts b/integ-tests/tui/add-gateway-jwt.test.ts index e18ea5e22..15e1e7284 100644 --- a/integ-tests/tui/add-gateway-jwt.test.ts +++ b/integ-tests/tui/add-gateway-jwt.test.ts @@ -133,9 +133,9 @@ describe('Add Gateway JWT Flow', () => { it('Step 1c: navigates to Gateway and enters the wizard', async () => { // Add Resource list order: - // 0: Agent, 1: Memory, 2: Identity, 3: Evaluator, - // 4: Online Eval Config, 5: Gateway, 6: Gateway Target - for (let i = 0; i < 5; i++) { + // 0: Harness, 1: Agent, 2: Memory, 3: Credential, 4: Evaluator, + // 5: Online Eval Config, 6: Gateway, 7: Gateway Target, 8: Policy + for (let i = 0; i < 6; i++) { await session.sendSpecialKey('down'); } await settle(); diff --git a/integ-tests/tui/add-memory-episodic.test.ts b/integ-tests/tui/add-memory-episodic.test.ts index c4dd65d46..c2caad335 100644 --- a/integ-tests/tui/add-memory-episodic.test.ts +++ b/integ-tests/tui/add-memory-episodic.test.ts @@ -94,7 +94,8 @@ describe('Add Memory with EPISODIC Strategy', () => { }); it('Step 3: selects Memory from the resource list', async () => { - // Add Resource list: 0: Agent, 1: Memory + // Add Resource list: 0: Harness, 1: Agent, 2: Memory + await session.sendSpecialKey('down'); await session.sendSpecialKey('down'); await settle(); diff --git a/integ-tests/tui/lifecycle-config.test.ts b/integ-tests/tui/lifecycle-config.test.ts index c8cc5947b..12e69e03c 100644 --- a/integ-tests/tui/lifecycle-config.test.ts +++ b/integ-tests/tui/lifecycle-config.test.ts @@ -252,7 +252,8 @@ describe('Add Agent BYO Flow: Lifecycle Configuration via TUI', () => { expect(atAdd).toBe(true); saveTextScreenshot(session, 'byo-02-add-resource'); - // Select Agent (first option) + // Select Agent (second option, after Harness) + await session.sendSpecialKey('down'); await session.sendSpecialKey('enter'); const atAgent = await safeWaitFor(session, /agent|Name/i, 5_000); diff --git a/package-lock.json b/package-lock.json index 45ad45ed3..e7c95133b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@aws/agentcore", - "version": "0.12.2", + "version": "1.0.0-preview.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@aws/agentcore", - "version": "0.12.2", + "version": "1.0.0-preview.6", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { @@ -24,7 +24,7 @@ "@aws-sdk/client-sts": "^3.893.0", "@aws-sdk/client-xray": "^3.1003.0", "@aws-sdk/credential-providers": "^3.893.0", - "@aws/agent-inspector": "0.2.1", + "@aws/agent-inspector": "0.3.0", "@commander-js/extra-typings": "^14.0.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-metrics-otlp-http": "^0.214.0", @@ -2903,9 +2903,9 @@ } }, "node_modules/@aws/agent-inspector": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/@aws/agent-inspector/-/agent-inspector-0.2.1.tgz", - "integrity": "sha512-kyL6RBcTj1hYIchtrHDlDyeqm2viVYMBxhZKVn8wJn058YhI52GIDuUFlKD1avd57X+LJKlHr5VcKvBZp7Sg6A==", + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/agent-inspector/-/agent-inspector-0.3.0.tgz", + "integrity": "sha512-xD7QPr1WWkT9QWRWo6e9kq8kYxJLQ8egGscgSZ6jCyW3wNV5fcQ6THcAR/71hxxMFF2aleNUc3D8MoqgiS4DVw==", "license": "Apache-2.0", "dependencies": { "@ag-ui/core": "^0.0.52", diff --git a/package.json b/package.json index dda0501e2..6433f329d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@aws/agentcore", - "version": "0.12.2", + "version": "1.0.0-preview.6", "description": "CLI for Amazon Bedrock AgentCore", "license": "Apache-2.0", "repository": { @@ -87,7 +87,7 @@ "@aws-sdk/client-sts": "^3.893.0", "@aws-sdk/client-xray": "^3.1003.0", "@aws-sdk/credential-providers": "^3.893.0", - "@aws/agent-inspector": "0.2.1", + "@aws/agent-inspector": "0.3.0", "@commander-js/extra-typings": "^14.0.0", "@opentelemetry/api": "^1.9.1", "@opentelemetry/exporter-metrics-otlp-http": "^0.214.0", diff --git a/schemas/agentcore.schema.v1.json b/schemas/agentcore.schema.v1.json index c2dd737a7..7434481f9 100644 --- a/schemas/agentcore.schema.v1.json +++ b/schemas/agentcore.schema.v1.json @@ -690,6 +690,10 @@ "type": "string", "minLength": 1 }, + "endpoint": { + "type": "string", + "minLength": 1 + }, "evaluators": { "minItems": 1, "type": "array", @@ -1840,6 +1844,330 @@ "required": ["name"], "additionalProperties": false } + }, + "harnesses": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 48, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]{0,47}$" + }, + "path": { + "type": "string", + "minLength": 1 + } + }, + "required": ["name", "path"], + "additionalProperties": false + } + }, + "configBundles": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 100, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]{0,99}$" + }, + "type": { + "default": "ConfigurationBundle", + "type": "string", + "const": "ConfigurationBundle" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 500 + }, + "components": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "configuration": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + "required": ["configuration"], + "additionalProperties": false + } + }, + "branchName": { + "type": "string", + "maxLength": 128 + }, + "commitMessage": { + "type": "string", + "maxLength": 500 + } + }, + "required": ["name", "components"], + "additionalProperties": false + } + }, + "abTests": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 48, + "pattern": "^[a-zA-Z][a-zA-Z0-9_]{0,47}$" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "mode": { + "default": "config-bundle", + "type": "string", + "enum": ["config-bundle", "target-based"] + }, + "gatewayRef": { + "type": "string", + "minLength": 1 + }, + "roleArn": { + "type": "string", + "minLength": 1 + }, + "variants": { + "minItems": 2, + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "enum": ["C", "T1"] + }, + "weight": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "variantConfiguration": { + "anyOf": [ + { + "type": "object", + "properties": { + "configurationBundle": { + "type": "object", + "properties": { + "bundleArn": { + "type": "string", + "minLength": 1 + }, + "bundleVersion": { + "type": "string", + "minLength": 1 + } + }, + "required": ["bundleArn", "bundleVersion"], + "additionalProperties": false + }, + "target": { + "not": {} + } + }, + "required": ["configurationBundle"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "configurationBundle": { + "not": {} + }, + "target": { + "type": "object", + "properties": { + "targetName": { + "type": "string", + "minLength": 1, + "maxLength": 100 + } + }, + "required": ["targetName"], + "additionalProperties": false + } + }, + "required": ["target"], + "additionalProperties": false + } + ] + } + }, + "required": ["name", "weight", "variantConfiguration"], + "additionalProperties": false + } + }, + "evaluationConfig": { + "anyOf": [ + { + "type": "object", + "properties": { + "onlineEvaluationConfigArn": { + "type": "string", + "minLength": 1 + } + }, + "required": ["onlineEvaluationConfigArn"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "perVariantOnlineEvaluationConfig": { + "minItems": 2, + "maxItems": 2, + "type": "array", + "items": { + "type": "object", + "properties": { + "treatmentName": { + "type": "string", + "enum": ["C", "T1"] + }, + "onlineEvaluationConfigArn": { + "type": "string", + "minLength": 1 + } + }, + "required": ["treatmentName", "onlineEvaluationConfigArn"], + "additionalProperties": false + } + } + }, + "required": ["perVariantOnlineEvaluationConfig"], + "additionalProperties": false + } + ] + }, + "gatewayFilter": { + "type": "object", + "properties": { + "targetPaths": { + "maxItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 500 + } + } + }, + "required": ["targetPaths"], + "additionalProperties": false + }, + "trafficAllocationConfig": { + "type": "object", + "properties": { + "routeOnHeader": { + "type": "object", + "properties": { + "headerName": { + "type": "string", + "minLength": 1 + } + }, + "required": ["headerName"], + "additionalProperties": false + } + }, + "required": ["routeOnHeader"], + "additionalProperties": false + }, + "maxDurationDays": { + "type": "integer", + "minimum": 1, + "maximum": 90 + }, + "enableOnCreate": { + "type": "boolean" + }, + "promoted": { + "type": "boolean" + } + }, + "required": ["name", "gatewayRef", "variants", "evaluationConfig"], + "additionalProperties": false + } + }, + "httpGateways": { + "default": [], + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 48, + "pattern": "^[a-zA-Z][a-zA-Z0-9-]{0,47}$" + }, + "description": { + "type": "string", + "minLength": 1, + "maxLength": 200 + }, + "runtimeRef": { + "type": "string", + "minLength": 1 + }, + "roleArn": { + "type": "string", + "minLength": 1 + }, + "targets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1, + "maxLength": 100 + }, + "runtimeRef": { + "type": "string", + "minLength": 1 + }, + "qualifier": { + "default": "DEFAULT", + "type": "string", + "minLength": 1 + } + }, + "required": ["name", "runtimeRef"], + "additionalProperties": false + } + } + }, + "required": ["name", "runtimeRef"], + "additionalProperties": false + } } }, "required": ["name", "version"], diff --git a/scripts/bundle.mjs b/scripts/bundle.mjs index 0afdcf93d..ec46df8cc 100644 --- a/scripts/bundle.mjs +++ b/scripts/bundle.mjs @@ -1,5 +1,5 @@ /** - * bundle.mjs — Single command to build CLI + CDK constructs into one tarball. + * bundle.mjs — Single command to build CLI + CDK constructs + frontend into one tarball. * * This is a testing-only workflow. It does NOT modify the default build or * deployment flow. The normal `npm run build` + `npm pack` pipeline is unchanged. @@ -9,12 +9,16 @@ * At `agentcore create` time, CDKRenderer detects this tarball and installs it * after the normal `npm install`, overriding the registry version. * + * It also builds the @aws/agent-inspector frontend and copies its dist-assets + * into the CLI's dist/agent-inspector/ directory, overriding the npm registry version. + * * Usage: * node scripts/bundle.mjs * npm run bundle * * Environment variables: - * AGENTCORE_CDK_PATH — absolute path to the agentcore-l3-cdk-constructs repo + * AGENTCORE_CDK_PATH — absolute path to the agentcore-l3-cdk-constructs repo + * AGENT_INSPECTOR_PATH — absolute path to the agent-inspector repo */ import { execFileSync } from 'node:child_process'; import * as fs from 'node:fs'; @@ -75,13 +79,38 @@ function resolveCdkPath() { return cloneDir; } +/** + * Resolve the agent-inspector repo path. Priority: + * 1. AGENT_INSPECTOR_PATH env var + * 2. Sibling directory ../agent-inspector + */ +function resolveInspectorPath() { + if (process.env.AGENT_INSPECTOR_PATH) { + const p = path.resolve(process.env.AGENT_INSPECTOR_PATH); + if (fs.existsSync(path.join(p, 'package.json'))) { + log(`Using agent-inspector from AGENT_INSPECTOR_PATH: ${p}`); + return p; + } + console.warn(` WARNING: AGENT_INSPECTOR_PATH=${p} does not contain package.json, ignoring.`); + } + + const sibling = path.resolve(cliRoot, '..', 'agent-inspector'); + if (fs.existsSync(path.join(sibling, 'package.json'))) { + log(`Using agent-inspector from sibling directory: ${sibling}`); + return sibling; + } + + return null; +} + // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- log('Starting bundle process...'); -const timestamp = Math.floor(Date.now() / 1000); +const now = new Date(); +const timestamp = now.toISOString().replace(/[-:T]/g, '').slice(0, 14); log(`Bundle timestamp: ${timestamp}`); // Helper to bump a package version with a unique e2e timestamp tag. @@ -91,7 +120,9 @@ function bumpVersion(pkgDir) { const pkg = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf8')); const originalVersion = pkg.version; const baseVersion = originalVersion.split('-')[0]; - pkg.version = `${baseVersion}-${timestamp}`; + const prerelease = originalVersion.includes('-') ? originalVersion.split('-').slice(1).join('-') : ''; + const tag = prerelease ? `${prerelease}-${timestamp}` : timestamp; + pkg.version = `${baseVersion}-${tag}`; fs.writeFileSync(pkgJsonPath, JSON.stringify(pkg, null, 2) + '\n'); log(`Bumped ${pkg.name} version: ${originalVersion} -> ${pkg.version}`); return { pkgJsonPath, originalVersion, bumpedVersion: pkg.version }; @@ -141,7 +172,33 @@ const bundledTarballDest = path.join(cliRoot, 'dist', 'assets', 'bundled-agentco fs.copyFileSync(cdkTarballSrc, bundledTarballDest); log(`Placed CDK tarball at ${bundledTarballDest}`); -// Step 5: Bump CLI version and pack into final tarball (includes the bundled CDK tarball) +// Step 5: Build and bundle agent-inspector frontend (overrides the npm version) +const inspectorPath = resolveInspectorPath(); +if (inspectorPath) { + log('Installing agent-inspector dependencies...'); + run('npm', ['install'], { cwd: inspectorPath }); + + log('Building agent-inspector...'); + run('npm', ['run', 'build'], { cwd: inspectorPath }); + + const inspectorDistSrc = path.join(inspectorPath, 'dist-assets'); + const inspectorDistDest = path.join(cliRoot, 'dist', 'agent-inspector'); + + if (fs.existsSync(inspectorDistSrc)) { + if (fs.existsSync(inspectorDistDest)) { + fs.rmSync(inspectorDistDest, { recursive: true }); + } + fs.cpSync(inspectorDistSrc, inspectorDistDest, { recursive: true }); + log(`Copied agent-inspector frontend to ${inspectorDistDest}`); + } else { + console.error(`ERROR: agent-inspector build did not produce dist-assets/ at ${inspectorDistSrc}`); + process.exit(1); + } +} else { + log('No local agent-inspector found — using npm registry version.'); +} + +// Step 6: Bump CLI version and pack into final tarball (includes the bundled CDK tarball + frontend) const cliVersionInfo = bumpVersion(cliRoot); try { log('Packing CLI tarball...'); diff --git a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap index 8fa17f318..30e9be0f4 100644 --- a/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap +++ b/src/assets/__tests__/__snapshots__/assets.snapshot.test.ts.snap @@ -99,6 +99,45 @@ async function main() { throw new Error('No deployment targets configured. Please define targets in agentcore/aws-targets.json'); } + // Read harness configs for role creation. + // Harness fields may not yet be on the AgentCoreProjectSpec type from @aws/agentcore-cdk, + // so we read them dynamically via specAny (same pattern as gateways above). + // Harness paths in agentcore.json are relative to the project root (parent of agentcore/). + const projectRoot = path.resolve(configRoot, '..'); + const harnessConfigs: { + name: string; + executionRoleArn?: string; + memoryName?: string; + containerUri?: string; + hasDockerfile?: boolean; + dockerfile?: string; + codeLocation?: string; + tools?: { type: string; name: string }[]; + apiKeyArn?: string; + }[] = []; + for (const entry of specAny.harnesses ?? []) { + const harnessDir = path.resolve(projectRoot, entry.path); + const harnessPath = path.resolve(harnessDir, 'harness.json'); + try { + const harnessSpec = JSON.parse(fs.readFileSync(harnessPath, 'utf-8')); + harnessConfigs.push({ + name: entry.name, + executionRoleArn: harnessSpec.executionRoleArn, + memoryName: harnessSpec.memory?.name, + containerUri: harnessSpec.containerUri, + hasDockerfile: !!harnessSpec.dockerfile, + dockerfile: harnessSpec.dockerfile, + codeLocation: harnessSpec.dockerfile ? harnessDir : undefined, + tools: harnessSpec.tools, + apiKeyArn: harnessSpec.model?.apiKeyArn, + }); + } catch (err) { + throw new Error( + \`Could not read harness.json for "\${entry.name}" at \${harnessPath}: \${err instanceof Error ? err.message : err}\` + ); + } + } + const app = new App(); for (const target of targets) { @@ -118,6 +157,7 @@ async function main() { spec, mcpSpec, credentials, + harnesses: harnessConfigs.length > 0 ? harnessConfigs : undefined, env, description: \`AgentCore stack for \${spec.name} deployed to \${target.name} (\${target.region})\`, tags: { @@ -265,6 +305,18 @@ exports[`Assets Directory Snapshots > CDK assets > cdk/cdk/lib/cdk-stack.ts shou import { CfnOutput, Stack, type StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; +export interface HarnessConfig { + name: string; + executionRoleArn?: string; + memoryName?: string; + containerUri?: string; + hasDockerfile?: boolean; + dockerfile?: string; + codeLocation?: string; + tools?: { type: string; name: string }[]; + apiKeyArn?: string; +} + export interface AgentCoreStackProps extends StackProps { /** * The AgentCore project specification containing agents, memories, and credentials. @@ -278,6 +330,14 @@ export interface AgentCoreStackProps extends StackProps { * Credential provider ARNs from deployed state, keyed by credential name. */ credentials?: Record; + /** + * Harness role configurations. Each entry creates an IAM execution role for a harness. + * + * When \`hasDockerfile\` is true and \`codeLocation\` is provided (without an explicit + * \`containerUri\`), the L3 construct builds and pushes a container image via CodeBuild + * and emits its URI as a stack output for the post-CDK harness deployer. + */ + harnesses?: HarnessConfig[]; } /** @@ -293,11 +353,12 @@ export class AgentCoreStack extends Stack { constructor(scope: Construct, id: string, props: AgentCoreStackProps) { super(scope, id, props); - const { spec, mcpSpec, credentials } = props; + const { spec, mcpSpec, credentials, harnesses } = props; - // Create AgentCoreApplication with all agents + // Create AgentCoreApplication with all agents and harness roles this.application = new AgentCoreApplication(this, 'Application', { spec, + harnesses: harnesses?.length ? harnesses : undefined, }); // Create AgentCoreMcp if there are gateways configured @@ -382,6 +443,7 @@ test('AgentCoreStack synthesizes with empty spec', () => { credentials: [], evaluators: [], onlineEvalConfigs: [], + configBundles: [], policyEngines: [], agentCoreGateways: [], mcpRuntimeTools: [], @@ -449,6 +511,7 @@ exports[`Assets Directory Snapshots > File listing > should match the expected f "evaluators/python-lambda/execution-role-policy.json", "evaluators/python-lambda/lambda_function.py", "evaluators/python-lambda/pyproject.toml", + "harness/invoke.py.template", "mcp/python-lambda/README.md", "mcp/python-lambda/handler.py", "mcp/python-lambda/pyproject.toml", @@ -3700,9 +3763,15 @@ Thumbs.db exports[`Assets Directory Snapshots > Python framework assets > python/python/http/langchain_langgraph/base/main.py should match snapshot 1`] = ` "import os -from langchain_core.messages import HumanMessage +from typing import Any + +from langchain_core.messages import HumanMessage{{#if hasConfigBundle}}, SystemMessage{{/if}} from langgraph.prebuilt import create_react_agent from langchain.tools import tool +{{#if hasConfigBundle}} +from langchain_core.callbacks import BaseCallbackHandler +from bedrock_agentcore.runtime.context import BedrockAgentCoreContext +{{/if}} from opentelemetry.instrumentation.langchain import LangchainInstrumentor from bedrock_agentcore.runtime import BedrockAgentCoreApp from model.load import load_model @@ -3726,6 +3795,14 @@ def get_or_create_model(): return _llm +DEFAULT_SYSTEM_PROMPT = """ +You are a helpful assistant. Use tools when appropriate. +{{#if sessionStorageMountPath}} +You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{/if}} +""" + + # Define a simple function tool @tool def add_numbers(a: int, b: int) -> int: @@ -3789,13 +3866,28 @@ def list_files(directory: str = "") -> str: tools.extend([file_read, file_write, list_files]) {{/if}} -SYSTEM_PROMPT = """ -You are a helpful assistant. Use tools when appropriate. -{{#if sessionStorageMountPath}} -You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. -{{/if}} -""" +{{#if hasConfigBundle}} + +class ConfigBundleCallback(BaseCallbackHandler): + """Injects config bundle values into LangGraph agent at runtime. + + BedrockAgentCoreContext.get_config_bundle() fetches the component configuration + for the current runtime ARN from the config bundle service. The SDK caches the + result and refreshes on bundle version changes. + """ + + def on_chain_start(self, serialized: dict, inputs: dict, **kwargs: Any) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + prompt = config.get("systemPrompt", DEFAULT_SYSTEM_PROMPT) + messages = inputs.get("messages", []) + if messages and isinstance(messages[0], SystemMessage): + messages[0] = SystemMessage(content=prompt) + else: + messages.insert(0, SystemMessage(content=prompt)) + inputs["messages"] = messages + +{{/if}} @app.entrypoint async def invoke(payload, context): @@ -3814,7 +3906,21 @@ async def invoke(payload, context): mcp_tools = await mcp_client.get_tools() # Define the agent using create_react_agent - graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=SYSTEM_PROMPT) +{{#if hasConfigBundle}} + graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=DEFAULT_SYSTEM_PROMPT) + callback = ConfigBundleCallback() + + # Process the user prompt + prompt = payload.get("prompt", "What can you help me with?") + log.info(f"Agent input: {prompt}") + + # Run the agent with config bundle callback + result = await graph.ainvoke( + {"messages": [HumanMessage(content=prompt)]}, + config={"callbacks": [callback]}, + ) +{{else}} + graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=DEFAULT_SYSTEM_PROMPT) # Process the user prompt prompt = payload.get("prompt", "What can you help me with?") @@ -3822,6 +3928,7 @@ async def invoke(payload, context): # Run the agent result = await graph.ainvoke({"messages": [HumanMessage(content=prompt)]}) +{{/if}} # Return result output = result["messages"][-1].content @@ -4596,7 +4703,13 @@ Thumbs.db" `; exports[`Assets Directory Snapshots > Python framework assets > python/python/http/strands/base/main.py should match snapshot 1`] = ` -"from strands import Agent, tool +"from typing import Any + +from strands import Agent, tool +{{#if hasConfigBundle}} +from strands.hooks import HookProvider, HookRegistry, BeforeInvocationEvent, BeforeToolCallEvent +from bedrock_agentcore.runtime.context import BedrockAgentCoreContext +{{/if}} from bedrock_agentcore.runtime import BedrockAgentCoreApp from model.load import load_model {{#if hasGateway}} @@ -4621,11 +4734,26 @@ mcp_clients = get_all_gateway_mcp_clients() mcp_clients = [get_streamable_http_mcp_client()] {{/if}} +DEFAULT_SYSTEM_PROMPT = """ +You are a helpful assistant. Use tools when appropriate. +{{#if sessionStorageMountPath}} +You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{/if}} +""" + +{{#if hasConfigBundle}} +DEFAULT_TOOL_DESC = "Return the sum of two numbers" +{{/if}} + # Define a collection of tools used by the model tools = [] # Define a simple function tool +{{#if hasConfigBundle}} +@tool(description=DEFAULT_TOOL_DESC) +{{else}} @tool +{{/if}} def add_numbers(a: int, b: int) -> int: """Return the sum of two numbers""" return a+b @@ -4689,12 +4817,39 @@ for mcp_client in mcp_clients: if mcp_client: tools.append(mcp_client) -SYSTEM_PROMPT = """ -You are a helpful assistant. Use tools when appropriate. -{{#if sessionStorageMountPath}} -You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{#if hasConfigBundle}} + +class ConfigBundleHook(HookProvider): + """Injects config bundle values (system prompt, tool descriptions) before each invocation. + + BedrockAgentCoreContext.get_config_bundle() fetches the component configuration + for the current runtime ARN from the config bundle service. The SDK caches the + result and refreshes on bundle version changes. + """ + + def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: + registry.add_callback(BeforeInvocationEvent, self._inject_system_prompt) + registry.add_callback(BeforeToolCallEvent, self._override_tool_desc) + + def _inject_system_prompt(self, event: BeforeInvocationEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + prompt = config.get("systemPrompt", DEFAULT_SYSTEM_PROMPT) + + if prompt != event.agent.system_prompt: + event.agent.system_prompt = prompt + + def _override_tool_desc(self, event: BeforeToolCallEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + tool_descs = config.get("toolDescriptions", {}) + + tool_name = event.tool_use["name"] + override = tool_descs.get(tool_name) + if override and event.selected_tool: + spec = event.selected_tool.tool_spec + if spec and "description" in spec: + spec["description"] = override + {{/if}} -""" {{#if hasMemory}} def agent_factory(): @@ -4706,13 +4861,23 @@ def agent_factory(): cache[key] = Agent( model=load_model(), session_manager=get_memory_session_manager(session_id, user_id), - system_prompt=SYSTEM_PROMPT, - tools=tools + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools{{#if hasConfigBundle}}, + hooks=[ConfigBundleHook()]{{/if}} ) return cache[key] return get_or_create_agent get_or_create_agent = agent_factory() {{else}} +{{#if hasConfigBundle}} +def create_agent(): + return Agent( + model=load_model(), + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools, + hooks=[ConfigBundleHook()], + ) +{{else}} _agent = None def get_or_create_agent(): @@ -4720,11 +4885,12 @@ def get_or_create_agent(): if _agent is None: _agent = Agent( model=load_model(), - system_prompt=SYSTEM_PROMPT, + system_prompt=DEFAULT_SYSTEM_PROMPT, tools=tools ) return _agent {{/if}} +{{/if}} @app.entrypoint @@ -4735,8 +4901,12 @@ async def invoke(payload, context): session_id = getattr(context, 'session_id', 'default-session') user_id = getattr(context, 'user_id', 'default-user') agent = get_or_create_agent(session_id, user_id) +{{else}} +{{#if hasConfigBundle}} + agent = create_agent() {{else}} agent = get_or_create_agent() +{{/if}} {{/if}} # Execute and format response diff --git a/src/assets/cdk/bin/cdk.ts b/src/assets/cdk/bin/cdk.ts index 7a78b71cd..1c010e19b 100644 --- a/src/assets/cdk/bin/cdk.ts +++ b/src/assets/cdk/bin/cdk.ts @@ -54,6 +54,45 @@ async function main() { throw new Error('No deployment targets configured. Please define targets in agentcore/aws-targets.json'); } + // Read harness configs for role creation. + // Harness fields may not yet be on the AgentCoreProjectSpec type from @aws/agentcore-cdk, + // so we read them dynamically via specAny (same pattern as gateways above). + // Harness paths in agentcore.json are relative to the project root (parent of agentcore/). + const projectRoot = path.resolve(configRoot, '..'); + const harnessConfigs: { + name: string; + executionRoleArn?: string; + memoryName?: string; + containerUri?: string; + hasDockerfile?: boolean; + dockerfile?: string; + codeLocation?: string; + tools?: { type: string; name: string }[]; + apiKeyArn?: string; + }[] = []; + for (const entry of specAny.harnesses ?? []) { + const harnessDir = path.resolve(projectRoot, entry.path); + const harnessPath = path.resolve(harnessDir, 'harness.json'); + try { + const harnessSpec = JSON.parse(fs.readFileSync(harnessPath, 'utf-8')); + harnessConfigs.push({ + name: entry.name, + executionRoleArn: harnessSpec.executionRoleArn, + memoryName: harnessSpec.memory?.name, + containerUri: harnessSpec.containerUri, + hasDockerfile: !!harnessSpec.dockerfile, + dockerfile: harnessSpec.dockerfile, + codeLocation: harnessSpec.dockerfile ? harnessDir : undefined, + tools: harnessSpec.tools, + apiKeyArn: harnessSpec.model?.apiKeyArn, + }); + } catch (err) { + throw new Error( + `Could not read harness.json for "${entry.name}" at ${harnessPath}: ${err instanceof Error ? err.message : err}` + ); + } + } + const app = new App(); for (const target of targets) { @@ -73,6 +112,7 @@ async function main() { spec, mcpSpec, credentials, + harnesses: harnessConfigs.length > 0 ? harnessConfigs : undefined, env, description: `AgentCore stack for ${spec.name} deployed to ${target.name} (${target.region})`, tags: { diff --git a/src/assets/cdk/lib/cdk-stack.ts b/src/assets/cdk/lib/cdk-stack.ts index a4d277821..a89efc850 100644 --- a/src/assets/cdk/lib/cdk-stack.ts +++ b/src/assets/cdk/lib/cdk-stack.ts @@ -7,6 +7,18 @@ import { import { CfnOutput, Stack, type StackProps } from 'aws-cdk-lib'; import { Construct } from 'constructs'; +export interface HarnessConfig { + name: string; + executionRoleArn?: string; + memoryName?: string; + containerUri?: string; + hasDockerfile?: boolean; + dockerfile?: string; + codeLocation?: string; + tools?: { type: string; name: string }[]; + apiKeyArn?: string; +} + export interface AgentCoreStackProps extends StackProps { /** * The AgentCore project specification containing agents, memories, and credentials. @@ -20,6 +32,14 @@ export interface AgentCoreStackProps extends StackProps { * Credential provider ARNs from deployed state, keyed by credential name. */ credentials?: Record; + /** + * Harness role configurations. Each entry creates an IAM execution role for a harness. + * + * When `hasDockerfile` is true and `codeLocation` is provided (without an explicit + * `containerUri`), the L3 construct builds and pushes a container image via CodeBuild + * and emits its URI as a stack output for the post-CDK harness deployer. + */ + harnesses?: HarnessConfig[]; } /** @@ -35,11 +55,12 @@ export class AgentCoreStack extends Stack { constructor(scope: Construct, id: string, props: AgentCoreStackProps) { super(scope, id, props); - const { spec, mcpSpec, credentials } = props; + const { spec, mcpSpec, credentials, harnesses } = props; - // Create AgentCoreApplication with all agents + // Create AgentCoreApplication with all agents and harness roles this.application = new AgentCoreApplication(this, 'Application', { spec, + harnesses: harnesses?.length ? harnesses : undefined, }); // Create AgentCoreMcp if there are gateways configured diff --git a/src/assets/cdk/test/cdk.test.ts b/src/assets/cdk/test/cdk.test.ts index df5c767f9..79282f729 100644 --- a/src/assets/cdk/test/cdk.test.ts +++ b/src/assets/cdk/test/cdk.test.ts @@ -14,6 +14,7 @@ test('AgentCoreStack synthesizes with empty spec', () => { credentials: [], evaluators: [], onlineEvalConfigs: [], + configBundles: [], policyEngines: [], agentCoreGateways: [], mcpRuntimeTools: [], diff --git a/src/assets/harness/invoke.py.template b/src/assets/harness/invoke.py.template new file mode 100644 index 000000000..cf2527f44 --- /dev/null +++ b/src/assets/harness/invoke.py.template @@ -0,0 +1,74 @@ +""" +Standalone invoke script for AgentCore Harness. +Generated by: agentcore create --with-invoke-script + +Usage: + pip install boto3 + export HARNESS_ARN="arn:aws:bedrock-agentcore:::harness/" + python invoke.py "Hello, what can you do?" + python invoke.py --raw-events "Hello" +""" + +import argparse +import json +import os +import sys +import uuid + +import boto3 + +# --- Configuration --- +HARNESS_ARN = os.environ.get("HARNESS_ARN", "{{HARNESS_ARN}}") +REGION = os.environ.get("AWS_REGION", "{{REGION}}") +SESSION_ID = os.environ.get("SESSION_ID", str(uuid.uuid4())) + +parser = argparse.ArgumentParser(description="Invoke an AgentCore Harness") +parser.add_argument("prompt", nargs="?", default="Hello!", help="Prompt to send to the agent") +parser.add_argument("--raw-events", action="store_true", help="Print raw streaming events as JSON") +parser.add_argument("--session-id", default=SESSION_ID, help="Session ID for conversation continuity") +args = parser.parse_args() + +client = boto3.client("bedrock-agentcore", region_name=REGION) + +response = client.invoke_harness( + harnessArn=HARNESS_ARN, + runtimeSessionId=args.session_id, + messages=[ + { + "role": "user", + "content": [{"text": args.prompt}], + } + ], +) + +for event in response["stream"]: + if args.raw_events: + print(json.dumps(event, default=str)) + else: + if "contentBlockStart" in event: + start = event["contentBlockStart"].get("start", {}) + if "toolUse" in start: + tool = start["toolUse"] + print(f"\n🔧 Tool: {tool.get('name', 'unknown')}", flush=True) + elif "contentBlockDelta" in event: + delta = event["contentBlockDelta"].get("delta", {}) + if "text" in delta: + print(delta["text"], end="", flush=True) + elif "messageStop" in event: + stop_reason = event["messageStop"].get("stopReason", "") + if stop_reason == "end_turn": + print() + elif "metadata" in event: + usage = event["metadata"].get("usage", {}) + metrics = event["metadata"].get("metrics", {}) + latency = metrics.get("latencyMs", 0) / 1000 + print( + f"\n⚡ {usage.get('inputTokens', 0)} in · " + f"{usage.get('outputTokens', 0)} out · " + f"{latency:.1f}s", + file=sys.stderr, + ) + elif "internalServerException" in event: + print(f"\nError: {event['internalServerException']}", file=sys.stderr) + +print(f"\n🔗 Session: {args.session_id}", file=sys.stderr) diff --git a/src/assets/python/http/langchain_langgraph/base/main.py b/src/assets/python/http/langchain_langgraph/base/main.py index dcb9eb13c..773253da0 100644 --- a/src/assets/python/http/langchain_langgraph/base/main.py +++ b/src/assets/python/http/langchain_langgraph/base/main.py @@ -1,7 +1,13 @@ import os -from langchain_core.messages import HumanMessage +from typing import Any + +from langchain_core.messages import HumanMessage{{#if hasConfigBundle}}, SystemMessage{{/if}} from langgraph.prebuilt import create_react_agent from langchain.tools import tool +{{#if hasConfigBundle}} +from langchain_core.callbacks import BaseCallbackHandler +from bedrock_agentcore.runtime.context import BedrockAgentCoreContext +{{/if}} from opentelemetry.instrumentation.langchain import LangchainInstrumentor from bedrock_agentcore.runtime import BedrockAgentCoreApp from model.load import load_model @@ -25,6 +31,14 @@ def get_or_create_model(): return _llm +DEFAULT_SYSTEM_PROMPT = """ +You are a helpful assistant. Use tools when appropriate. +{{#if sessionStorageMountPath}} +You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{/if}} +""" + + # Define a simple function tool @tool def add_numbers(a: int, b: int) -> int: @@ -88,13 +102,28 @@ def list_files(directory: str = "") -> str: tools.extend([file_read, file_write, list_files]) {{/if}} -SYSTEM_PROMPT = """ -You are a helpful assistant. Use tools when appropriate. -{{#if sessionStorageMountPath}} -You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. -{{/if}} -""" +{{#if hasConfigBundle}} + +class ConfigBundleCallback(BaseCallbackHandler): + """Injects config bundle values into LangGraph agent at runtime. + + BedrockAgentCoreContext.get_config_bundle() fetches the component configuration + for the current runtime ARN from the config bundle service. The SDK caches the + result and refreshes on bundle version changes. + """ + + def on_chain_start(self, serialized: dict, inputs: dict, **kwargs: Any) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + prompt = config.get("systemPrompt", DEFAULT_SYSTEM_PROMPT) + messages = inputs.get("messages", []) + if messages and isinstance(messages[0], SystemMessage): + messages[0] = SystemMessage(content=prompt) + else: + messages.insert(0, SystemMessage(content=prompt)) + inputs["messages"] = messages + +{{/if}} @app.entrypoint async def invoke(payload, context): @@ -113,7 +142,21 @@ async def invoke(payload, context): mcp_tools = await mcp_client.get_tools() # Define the agent using create_react_agent - graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=SYSTEM_PROMPT) +{{#if hasConfigBundle}} + graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=DEFAULT_SYSTEM_PROMPT) + callback = ConfigBundleCallback() + + # Process the user prompt + prompt = payload.get("prompt", "What can you help me with?") + log.info(f"Agent input: {prompt}") + + # Run the agent with config bundle callback + result = await graph.ainvoke( + {"messages": [HumanMessage(content=prompt)]}, + config={"callbacks": [callback]}, + ) +{{else}} + graph = create_react_agent(get_or_create_model(), tools=mcp_tools + tools, prompt=DEFAULT_SYSTEM_PROMPT) # Process the user prompt prompt = payload.get("prompt", "What can you help me with?") @@ -121,6 +164,7 @@ async def invoke(payload, context): # Run the agent result = await graph.ainvoke({"messages": [HumanMessage(content=prompt)]}) +{{/if}} # Return result output = result["messages"][-1].content diff --git a/src/assets/python/http/strands/base/main.py b/src/assets/python/http/strands/base/main.py index f7b69d3e4..0cc8771ad 100644 --- a/src/assets/python/http/strands/base/main.py +++ b/src/assets/python/http/strands/base/main.py @@ -1,4 +1,10 @@ +from typing import Any + from strands import Agent, tool +{{#if hasConfigBundle}} +from strands.hooks import HookProvider, HookRegistry, BeforeInvocationEvent, BeforeToolCallEvent +from bedrock_agentcore.runtime.context import BedrockAgentCoreContext +{{/if}} from bedrock_agentcore.runtime import BedrockAgentCoreApp from model.load import load_model {{#if hasGateway}} @@ -23,11 +29,26 @@ mcp_clients = [get_streamable_http_mcp_client()] {{/if}} +DEFAULT_SYSTEM_PROMPT = """ +You are a helpful assistant. Use tools when appropriate. +{{#if sessionStorageMountPath}} +You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{/if}} +""" + +{{#if hasConfigBundle}} +DEFAULT_TOOL_DESC = "Return the sum of two numbers" +{{/if}} + # Define a collection of tools used by the model tools = [] # Define a simple function tool +{{#if hasConfigBundle}} +@tool(description=DEFAULT_TOOL_DESC) +{{else}} @tool +{{/if}} def add_numbers(a: int, b: int) -> int: """Return the sum of two numbers""" return a+b @@ -91,12 +112,39 @@ def list_files(directory: str = "") -> str: if mcp_client: tools.append(mcp_client) -SYSTEM_PROMPT = """ -You are a helpful assistant. Use tools when appropriate. -{{#if sessionStorageMountPath}} -You have persistent storage at {{sessionStorageMountPath}}. Use file tools to read and write files. Data persists across sessions. +{{#if hasConfigBundle}} + +class ConfigBundleHook(HookProvider): + """Injects config bundle values (system prompt, tool descriptions) before each invocation. + + BedrockAgentCoreContext.get_config_bundle() fetches the component configuration + for the current runtime ARN from the config bundle service. The SDK caches the + result and refreshes on bundle version changes. + """ + + def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: + registry.add_callback(BeforeInvocationEvent, self._inject_system_prompt) + registry.add_callback(BeforeToolCallEvent, self._override_tool_desc) + + def _inject_system_prompt(self, event: BeforeInvocationEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + prompt = config.get("systemPrompt", DEFAULT_SYSTEM_PROMPT) + + if prompt != event.agent.system_prompt: + event.agent.system_prompt = prompt + + def _override_tool_desc(self, event: BeforeToolCallEvent) -> None: + config = BedrockAgentCoreContext.get_config_bundle() + tool_descs = config.get("toolDescriptions", {}) + + tool_name = event.tool_use["name"] + override = tool_descs.get(tool_name) + if override and event.selected_tool: + spec = event.selected_tool.tool_spec + if spec and "description" in spec: + spec["description"] = override + {{/if}} -""" {{#if hasMemory}} def agent_factory(): @@ -108,13 +156,23 @@ def get_or_create_agent(session_id, user_id): cache[key] = Agent( model=load_model(), session_manager=get_memory_session_manager(session_id, user_id), - system_prompt=SYSTEM_PROMPT, - tools=tools + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools{{#if hasConfigBundle}}, + hooks=[ConfigBundleHook()]{{/if}} ) return cache[key] return get_or_create_agent get_or_create_agent = agent_factory() {{else}} +{{#if hasConfigBundle}} +def create_agent(): + return Agent( + model=load_model(), + system_prompt=DEFAULT_SYSTEM_PROMPT, + tools=tools, + hooks=[ConfigBundleHook()], + ) +{{else}} _agent = None def get_or_create_agent(): @@ -122,11 +180,12 @@ def get_or_create_agent(): if _agent is None: _agent = Agent( model=load_model(), - system_prompt=SYSTEM_PROMPT, + system_prompt=DEFAULT_SYSTEM_PROMPT, tools=tools ) return _agent {{/if}} +{{/if}} @app.entrypoint @@ -137,8 +196,12 @@ async def invoke(payload, context): session_id = getattr(context, 'session_id', 'default-session') user_id = getattr(context, 'user_id', 'default-user') agent = get_or_create_agent(session_id, user_id) +{{else}} +{{#if hasConfigBundle}} + agent = create_agent() {{else}} agent = get_or_create_agent() +{{/if}} {{/if}} # Execute and format response diff --git a/src/cli/__tests__/global-config.test.ts b/src/cli/__tests__/global-config.test.ts index 2851a13a4..6e2038973 100644 --- a/src/cli/__tests__/global-config.test.ts +++ b/src/cli/__tests__/global-config.test.ts @@ -1,4 +1,9 @@ -import { getOrCreateInstallationId, readGlobalConfig, updateGlobalConfig } from '../global-config'; +import { + getOrCreateInstallationId, + readGlobalConfig, + readGlobalConfigSync, + updateGlobalConfig, +} from '../../lib/schemas/io/global-config'; import { createTempConfig } from './helpers/temp-config'; import { readFile, writeFile } from 'fs/promises'; import { afterAll, beforeEach, describe, expect, it } from 'vitest'; @@ -21,10 +26,29 @@ describe('global-config', () => { it('returns empty object when file is missing or invalid', async () => { expect(await readGlobalConfig(tmp.testDir + '/nonexistent.json')).toEqual({}); - await writeFile(tmp.configFile, JSON.stringify({ telemetry: { enabled: 'false' } })); + await writeFile(tmp.configFile, 'not json'); expect(await readGlobalConfig(tmp.configFile)).toEqual({}); }); + it('drops invalid fields while preserving valid ones', async () => { + await writeFile( + tmp.configFile, + JSON.stringify({ + transactionSearchIndexPercentage: 'not-a-number', + uvIndex: 'https://valid.url', + telemetry: { enabled: 'yes', endpoint: 'https://example.com' }, + }) + ); + + const config = await readGlobalConfig(tmp.configFile); + + expect(config).toEqual({ + transactionSearchIndexPercentage: undefined, + uvIndex: 'https://valid.url', + telemetry: { enabled: undefined, endpoint: 'https://example.com' }, + }); + }); + it('preserves unknown fields via passthrough', async () => { const full = { installationId: 'abc-123', @@ -39,6 +63,21 @@ describe('global-config', () => { }); }); + describe('readGlobalConfigSync', () => { + it('returns parsed config when file exists', async () => { + await writeFile(tmp.configFile, JSON.stringify({ telemetry: { enabled: false } })); + + expect(readGlobalConfigSync(tmp.configFile)).toEqual({ telemetry: { enabled: false } }); + }); + + it('returns empty object when file is missing or invalid', async () => { + expect(readGlobalConfigSync(tmp.testDir + '/nonexistent.json')).toEqual({}); + + await writeFile(tmp.configFile, 'not json'); + expect(readGlobalConfigSync(tmp.configFile)).toEqual({}); + }); + }); + describe('updateGlobalConfig', () => { it('creates directory and writes config when none exists', async () => { const fresh = createTempConfig('gc-fresh'); diff --git a/src/cli/__tests__/update-notifier.test.ts b/src/cli/__tests__/update-notifier.test.ts index 3713e51f2..27eb3f649 100644 --- a/src/cli/__tests__/update-notifier.test.ts +++ b/src/cli/__tests__/update-notifier.test.ts @@ -15,6 +15,12 @@ vi.mock('fs/promises', () => ({ vi.mock('../constants.js', () => ({ PACKAGE_VERSION: '1.0.0', + getDistroConfig: () => ({ + packageName: '@aws/agentcore', + registryUrl: 'https://registry.npmjs.org', + distTag: 'latest', + installCommand: 'npm install -g @aws/agentcore@latest', + }), })); const { mockFetchLatestVersion, mockCompareVersions } = vi.hoisted(() => ({ diff --git a/src/cli/aws/__tests__/agentcore-ab-tests.test.ts b/src/cli/aws/__tests__/agentcore-ab-tests.test.ts new file mode 100644 index 000000000..94dca3bdb --- /dev/null +++ b/src/cli/aws/__tests__/agentcore-ab-tests.test.ts @@ -0,0 +1,345 @@ +import { createABTest, deleteABTest, getABTest, listABTests, updateABTest } from '../agentcore-ab-tests.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +vi.mock('../account', () => ({ + getCredentialProvider: vi.fn().mockReturnValue({ + accessKeyId: 'AKID', + secretAccessKey: 'SECRET', + sessionToken: 'TOKEN', + }), +})); + +vi.mock('@smithy/signature-v4', () => ({ + SignatureV4: class { + // eslint-disable-next-line @typescript-eslint/require-await + async sign(request: { headers: Record }) { + return { headers: { ...request.headers, Authorization: 'signed' } }; + } + }, +})); + +vi.mock('@aws-crypto/sha256-js', () => ({ + Sha256: class {}, +})); + +vi.mock('@aws-sdk/credential-provider-node', () => ({ + defaultProvider: vi.fn(), +})); + +function mockJsonResponse(body: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + }; +} + +describe('agentcore-ab-tests', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('createABTest', () => { + it('sends POST to /ab-tests with correct body', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-001', + abTestArn: 'arn:abt:001', + name: 'MyTest', + status: 'CREATED', + executionStatus: 'STOPPED', + createdAt: '2026-01-01T00:00:00Z', + }) + ); + + const result = await createABTest({ + region: 'us-east-1', + name: 'MyTest', + gatewayArn: 'arn:aws:bedrock-agentcore:us-east-1:123:gateway/gw-1', + roleArn: 'arn:aws:iam::123:role/TestRole', + variants: [ + { + name: 'C', + weight: 80, + variantConfiguration: { configurationBundle: { bundleArn: 'arn:bundle:c', bundleVersion: 'v1' } }, + }, + { + name: 'T1', + weight: 20, + variantConfiguration: { configurationBundle: { bundleArn: 'arn:bundle:t', bundleVersion: 'v1' } }, + }, + ], + evaluationConfig: { onlineEvaluationConfigArn: 'arn:eval:config' }, + }); + + expect(result.abTestId).toBe('abt-001'); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/ab-tests'), + expect.objectContaining({ method: 'POST' }) + ); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.name).toBe('MyTest'); + expect(body.gatewayArn).toBe('arn:aws:bedrock-agentcore:us-east-1:123:gateway/gw-1'); + expect(body.variants).toHaveLength(2); + expect(body.clientToken).toBeDefined(); + }); + + it('omits optional fields when not provided', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-002', + abTestArn: 'arn:abt:002', + status: 'CREATED', + executionStatus: 'STOPPED', + createdAt: '2026-01-01T00:00:00Z', + }) + ); + + await createABTest({ + region: 'us-east-1', + name: 'Test', + gatewayArn: 'arn:gw', + roleArn: 'arn:role', + variants: [], + evaluationConfig: { onlineEvaluationConfigArn: 'arn:eval' }, + }); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.description).toBeUndefined(); + expect(body.trafficAllocationConfig).toBeUndefined(); + expect(body.maxDurationDays).toBeUndefined(); + expect(body.enableOnCreate).toBeUndefined(); + }); + + it('includes optional fields when provided', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-003', + abTestArn: 'arn:abt:003', + status: 'CREATED', + executionStatus: 'RUNNING', + createdAt: '2026-01-01T00:00:00Z', + }) + ); + + await createABTest({ + region: 'us-east-1', + name: 'Test', + description: 'A description', + gatewayArn: 'arn:gw', + roleArn: 'arn:role', + variants: [], + evaluationConfig: { onlineEvaluationConfigArn: 'arn:eval' }, + trafficAllocationConfig: { routeOnHeader: { headerName: 'X-AB' } }, + maxDurationDays: 30, + enableOnCreate: true, + }); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.description).toBe('A description'); + expect(body.trafficAllocationConfig).toEqual({ routeOnHeader: { headerName: 'X-AB' } }); + expect(body.maxDurationDays).toBe(30); + expect(body.enableOnCreate).toBe(true); + }); + + it('throws on non-ok response', async () => { + mockFetch.mockResolvedValue({ + ok: false, + status: 400, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('Bad Request'), + }); + + await expect( + createABTest({ + region: 'us-east-1', + name: 'Test', + gatewayArn: 'arn:gw', + roleArn: 'arn:role', + variants: [], + evaluationConfig: { onlineEvaluationConfigArn: 'arn:eval' }, + }) + ).rejects.toThrow('ABTest API error (400)'); + }); + }); + + describe('getABTest', () => { + it('sends GET to /ab-tests/{id}', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-123', + abTestArn: 'arn:abt:123', + name: 'MyTest', + status: 'ACTIVE', + executionStatus: 'RUNNING', + gatewayArn: 'arn:gw', + roleArn: 'arn:role', + variants: [], + evaluationConfig: { onlineEvaluationConfigArn: 'arn:eval' }, + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-02T00:00:00Z', + results: { + analysisTimestamp: '2026-01-02T00:00:00Z', + evaluatorMetrics: [], + }, + }) + ); + + const result = await getABTest({ region: 'us-east-1', abTestId: 'abt-123' }); + + expect(result.abTestId).toBe('abt-123'); + expect(result.results).toBeDefined(); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/ab-tests/abt-123'), + expect.objectContaining({ method: 'GET' }) + ); + }); + }); + + describe('updateABTest', () => { + it('sends PUT to /ab-tests/{id} with only defined fields', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-123', + abTestArn: 'arn:abt:123', + status: 'ACTIVE', + executionStatus: 'PAUSED', + updatedAt: '2026-01-02T00:00:00Z', + }) + ); + + await updateABTest({ + region: 'us-east-1', + abTestId: 'abt-123', + executionStatus: 'PAUSED', + }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/ab-tests/abt-123'), + expect.objectContaining({ method: 'PUT' }) + ); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.executionStatus).toBe('PAUSED'); + expect(body.clientToken).toBeDefined(); + expect(body.name).toBeUndefined(); + expect(body.description).toBeUndefined(); + expect(body.variants).toBeUndefined(); + }); + + it('includes all provided fields', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTestId: 'abt-123', + abTestArn: 'arn:abt:123', + status: 'ACTIVE', + executionStatus: 'RUNNING', + updatedAt: '2026-01-02T00:00:00Z', + }) + ); + + await updateABTest({ + region: 'us-east-1', + abTestId: 'abt-123', + name: 'Updated', + description: 'New desc', + maxDurationDays: 60, + roleArn: 'arn:new-role', + }); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.name).toBe('Updated'); + expect(body.description).toBe('New desc'); + expect(body.maxDurationDays).toBe(60); + expect(body.roleArn).toBe('arn:new-role'); + }); + }); + + describe('deleteABTest', () => { + it('sends DELETE to /ab-tests/{id} and returns success', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({}, 204)); + + const result = await deleteABTest({ region: 'us-east-1', abTestId: 'abt-123' }); + + expect(result.success).toBe(true); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/ab-tests/abt-123'), + expect.objectContaining({ method: 'DELETE' }) + ); + }); + + it('returns error on 404', async () => { + mockFetch.mockResolvedValue({ + ok: false, + status: 404, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('Not Found'), + }); + + const result = await deleteABTest({ region: 'us-east-1', abTestId: 'abt-999' }); + + expect(mockFetch).toHaveBeenCalledTimes(1); + expect(mockFetch.mock.calls[0]![0]).toContain('/ab-tests/abt-999'); + expect(result.success).toBe(false); + expect(result.error).toContain('ABTest API error (404)'); + }); + + it('returns error on network failure', async () => { + mockFetch.mockRejectedValue(new Error('Network error')); + + const result = await deleteABTest({ region: 'us-east-1', abTestId: 'abt-123' }); + + expect(result.success).toBe(false); + expect(result.error).toBe('Network error'); + }); + }); + + describe('listABTests', () => { + it('sends GET to /ab-tests', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + abTests: [ + { + abTestId: 'abt-1', + abTestArn: 'arn:abt:1', + name: 'Test1', + status: 'ACTIVE', + executionStatus: 'RUNNING', + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-01T00:00:00Z', + }, + ], + }) + ); + + const result = await listABTests({ region: 'us-east-1' }); + + expect(result.abTests).toHaveLength(1); + expect(result.abTests[0]!.name).toBe('Test1'); + }); + + it('passes maxResults and nextToken as query params', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({ abTests: [] })); + + await listABTests({ region: 'us-east-1', maxResults: 10, nextToken: 'abc' }); + + const url = mockFetch.mock.calls[0]![0] as string; + expect(url).toContain('maxResults=10'); + expect(url).toContain('nextToken=abc'); + }); + + it('returns empty array when response has no abTests', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({})); + + const result = await listABTests({ region: 'us-east-1' }); + + expect(result.abTests).toEqual([]); + }); + }); +}); diff --git a/src/cli/aws/__tests__/agentcore-harness.test.ts b/src/cli/aws/__tests__/agentcore-harness.test.ts new file mode 100644 index 000000000..7d14d776c --- /dev/null +++ b/src/cli/aws/__tests__/agentcore-harness.test.ts @@ -0,0 +1,451 @@ +import { + createHarness, + deleteHarness, + getHarness, + invokeHarness, + listAllHarnesses, + listHarnesses, + updateHarness, +} from '../agentcore-harness.js'; +import { EventStreamCodec } from '@smithy/eventstream-codec'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockRequest, mockRequestRaw } = vi.hoisted(() => ({ + mockRequest: vi.fn(), + mockRequestRaw: vi.fn(), +})); + +vi.mock('../api-client', () => ({ + AgentCoreApiClient: class { + request = mockRequest; + requestRaw = mockRequestRaw; + }, + AgentCoreApiError: class extends Error { + statusCode: number; + requestId: string | undefined; + errorBody: string; + constructor(statusCode: number, errorBody: string, requestId?: string) { + super(`AgentCore API error (${statusCode}): ${errorBody}`); + this.statusCode = statusCode; + this.requestId = requestId; + this.errorBody = errorBody; + } + }, +})); + +describe('Harness control plane operations', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('createHarness', () => { + it('sends POST /harnesses with correct body', async () => { + const harness = { harnessId: 'h-123', harnessName: 'test', status: 'CREATING' }; + mockRequest.mockResolvedValue({ harness }); + + const result = await createHarness({ + region: 'us-west-2', + harnessName: 'test', + executionRoleArn: 'arn:aws:iam::123:role/TestRole', + model: { bedrockModelConfig: { modelId: 'us.anthropic.claude-sonnet-4-6-20250514-v1:0' } }, + systemPrompt: [{ text: 'You are helpful.' }], + tools: [{ type: 'agentcore_browser', name: 'browser' }], + maxIterations: 75, + }); + + expect(result.harness.harnessId).toBe('h-123'); + expect(mockRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'POST', + path: '/harnesses', + body: expect.objectContaining({ + harnessName: 'test', + executionRoleArn: 'arn:aws:iam::123:role/TestRole', + clientToken: expect.any(String), + model: { bedrockModelConfig: { modelId: 'us.anthropic.claude-sonnet-4-6-20250514-v1:0' } }, + systemPrompt: [{ text: 'You are helpful.' }], + tools: [{ type: 'agentcore_browser', name: 'browser' }], + maxIterations: 75, + }), + }) + ); + }); + + it('omits optional fields when not provided', async () => { + mockRequest.mockResolvedValue({ harness: { harnessId: 'h-1' } }); + + await createHarness({ + region: 'us-west-2', + harnessName: 'minimal', + executionRoleArn: 'arn:aws:iam::123:role/R', + }); + + const body = mockRequest.mock.calls[0]![0].body; + expect(body.model).toBeUndefined(); + expect(body.tools).toBeUndefined(); + expect(body.memory).toBeUndefined(); + expect(body.maxIterations).toBeUndefined(); + }); + }); + + describe('getHarness', () => { + it('sends GET /harnesses/{harnessId}', async () => { + const harness = { harnessId: 'h-123', status: 'READY' }; + mockRequest.mockResolvedValue({ harness }); + + const result = await getHarness({ region: 'us-west-2', harnessId: 'h-123' }); + + expect(result.harness.status).toBe('READY'); + expect(mockRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'GET', + path: '/harnesses/h-123', + }) + ); + }); + }); + + describe('updateHarness', () => { + it('sends PATCH /harnesses/{harnessId}', async () => { + mockRequest.mockResolvedValue({ harness: { harnessId: 'h-123', status: 'UPDATING' } }); + + await updateHarness({ + region: 'us-west-2', + harnessId: 'h-123', + model: { bedrockModelConfig: { modelId: 'new-model' } }, + maxTokens: 4096, + }); + + expect(mockRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'PATCH', + path: '/harnesses/h-123', + body: expect.objectContaining({ + clientToken: expect.any(String), + model: { bedrockModelConfig: { modelId: 'new-model' } }, + maxTokens: 4096, + }), + }) + ); + }); + + it('passes nullable wrapper fields for memory and environmentArtifact', async () => { + mockRequest.mockResolvedValue({ harness: { harnessId: 'h-123' } }); + + await updateHarness({ + region: 'us-west-2', + harnessId: 'h-123', + memory: { optionalValue: null }, + environmentArtifact: { optionalValue: null }, + }); + + const body = mockRequest.mock.calls[0]![0].body; + expect(body.memory).toEqual({ optionalValue: null }); + expect(body.environmentArtifact).toEqual({ optionalValue: null }); + }); + }); + + describe('deleteHarness', () => { + it('sends DELETE /harnesses/{harnessId} with clientToken query param', async () => { + mockRequest.mockResolvedValue({ harness: { harnessId: 'h-123', status: 'DELETING' } }); + + await deleteHarness({ region: 'us-west-2', harnessId: 'h-123' }); + + expect(mockRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'DELETE', + path: '/harnesses/h-123', + query: { clientToken: expect.any(String) }, + }) + ); + }); + }); + + describe('listHarnesses', () => { + it('sends GET /harnesses with query params', async () => { + mockRequest.mockResolvedValue({ + harnesses: [{ harnessId: 'h-1', harnessName: 'one' }], + nextToken: undefined, + }); + + const result = await listHarnesses({ region: 'us-west-2', maxResults: 10 }); + + expect(result.harnesses).toHaveLength(1); + expect(mockRequest).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'GET', + path: '/harnesses', + query: { maxResults: '10' }, + }) + ); + }); + }); + + describe('listAllHarnesses', () => { + it('auto-paginates across multiple pages', async () => { + mockRequest + .mockResolvedValueOnce({ + harnesses: [{ harnessId: 'h-1' }], + nextToken: 'tok-1', + }) + .mockResolvedValueOnce({ + harnesses: [{ harnessId: 'h-2' }], + nextToken: undefined, + }); + + const all = await listAllHarnesses('us-west-2'); + + expect(all).toHaveLength(2); + expect(all[0]!.harnessId).toBe('h-1'); + expect(all[1]!.harnessId).toBe('h-2'); + expect(mockRequest).toHaveBeenCalledTimes(2); + }); + }); +}); + +describe('invokeHarness (streaming)', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const toUtf8 = (input: Uint8Array) => new TextDecoder().decode(input); + const fromUtf8 = (input: string) => new TextEncoder().encode(input); + const codec = new EventStreamCodec(toUtf8, fromUtf8); + + function encodeEvent(eventType: string, payload: Record): Uint8Array { + return codec.encode({ + headers: { + ':event-type': { type: 'string', value: eventType }, + ':content-type': { type: 'string', value: 'application/json' }, + ':message-type': { type: 'string', value: 'event' }, + }, + body: fromUtf8(JSON.stringify(payload)), + }); + } + + function makeStreamResponse(frames: Uint8Array[]): Response { + let totalLen = 0; + for (const f of frames) totalLen += f.length; + const combined = new Uint8Array(totalLen); + let off = 0; + for (const f of frames) { + combined.set(f, off); + off += f.length; + } + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(combined); + controller.close(); + }, + }); + return new Response(stream, { status: 200 }); + } + + it('yields messageStart events', async () => { + mockRequestRaw.mockResolvedValue(makeStreamResponse([encodeEvent('messageStart', { role: 'assistant' })])); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:aws:bedrock-agentcore:us-west-2:123:harness/h-123', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hello' }] }], + })) { + events.push(event); + } + + expect(events).toHaveLength(1); + expect(events[0]).toEqual({ type: 'messageStart', role: 'assistant' }); + }); + + it('yields text deltas', async () => { + mockRequestRaw.mockResolvedValue( + makeStreamResponse([ + encodeEvent('contentBlockDelta', { contentBlockIndex: 0, delta: { text: 'Hello' } }), + encodeEvent('contentBlockDelta', { contentBlockIndex: 0, delta: { text: ' world' } }), + ]) + ); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + })) { + events.push(event); + } + + expect(events).toHaveLength(2); + expect(events[0]).toEqual({ + type: 'contentBlockDelta', + contentBlockIndex: 0, + delta: { type: 'text', text: 'Hello' }, + }); + expect(events[1]).toEqual({ + type: 'contentBlockDelta', + contentBlockIndex: 0, + delta: { type: 'text', text: ' world' }, + }); + }); + + it('yields tool use start events', async () => { + mockRequestRaw.mockResolvedValue( + makeStreamResponse([ + encodeEvent('contentBlockStart', { + contentBlockIndex: 1, + start: { toolUse: { toolUseId: 'tu-1', name: 'exa_search', type: 'remote_mcp', serverName: 'exa' } }, + }), + ]) + ); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'search' }] }], + })) { + events.push(event); + } + + expect(events).toHaveLength(1); + expect(events[0]).toEqual({ + type: 'contentBlockStart', + contentBlockIndex: 1, + start: { + type: 'toolUse', + toolUse: { toolUseId: 'tu-1', name: 'exa_search', type: 'remote_mcp', serverName: 'exa' }, + }, + }); + }); + + it('yields messageStop with stopReason', async () => { + mockRequestRaw.mockResolvedValue(makeStreamResponse([encodeEvent('messageStop', { stopReason: 'end_turn' })])); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + })) { + events.push(event); + } + + expect(events[0]).toEqual({ type: 'messageStop', stopReason: 'end_turn' }); + }); + + it('yields metadata with token usage', async () => { + mockRequestRaw.mockResolvedValue( + makeStreamResponse([ + encodeEvent('metadata', { + usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + metrics: { latencyMs: 1200 }, + }), + ]) + ); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + })) { + events.push(event); + } + + expect(events[0]).toEqual({ + type: 'metadata', + usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 }, + metrics: { latencyMs: 1200 }, + }); + }); + + it('yields error events for exception event types', async () => { + mockRequestRaw.mockResolvedValue( + makeStreamResponse([encodeEvent('internalServerException', { message: 'Something broke' })]) + ); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + })) { + events.push(event); + } + + expect(events[0]).toEqual({ + type: 'error', + errorType: 'internalServerException', + message: 'Something broke', + }); + }); + + it('passes override options in request body', async () => { + mockRequestRaw.mockResolvedValue(makeStreamResponse([])); + + for await (const _event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + model: { bedrockModelConfig: { modelId: 'override-model' } }, + maxIterations: 20, + skills: [{ path: './skills/research' }], + })) { + // drain + } + + expect(mockRequestRaw).toHaveBeenCalledWith( + expect.objectContaining({ + method: 'POST', + path: '/harnesses/invoke', + query: { harnessArn: 'arn:harness' }, + headers: { 'X-Amzn-Bedrock-AgentCore-Runtime-Session-Id': 'sess-1' }, + body: expect.objectContaining({ + model: { bedrockModelConfig: { modelId: 'override-model' } }, + maxIterations: 20, + skills: [{ path: './skills/research' }], + }), + }) + ); + }); + + it('handles multiple event types in sequence', async () => { + mockRequestRaw.mockResolvedValue( + makeStreamResponse([ + encodeEvent('messageStart', { role: 'assistant' }), + encodeEvent('contentBlockDelta', { contentBlockIndex: 0, delta: { text: 'Hi' } }), + encodeEvent('contentBlockStop', { contentBlockIndex: 0 }), + encodeEvent('messageStop', { stopReason: 'end_turn' }), + encodeEvent('metadata', { + usage: { inputTokens: 10, outputTokens: 1, totalTokens: 11 }, + metrics: { latencyMs: 100 }, + }), + ]) + ); + + const events = []; + for await (const event of invokeHarness({ + region: 'us-west-2', + harnessArn: 'arn:harness', + runtimeSessionId: 'sess-1', + messages: [{ role: 'user', content: [{ text: 'hi' }] }], + })) { + events.push(event); + } + + expect(events).toHaveLength(5); + expect(events.map(e => e.type)).toEqual([ + 'messageStart', + 'contentBlockDelta', + 'contentBlockStop', + 'messageStop', + 'metadata', + ]); + }); +}); diff --git a/src/cli/aws/__tests__/agentcore-http-gateways.test.ts b/src/cli/aws/__tests__/agentcore-http-gateways.test.ts new file mode 100644 index 000000000..f9ace9a7a --- /dev/null +++ b/src/cli/aws/__tests__/agentcore-http-gateways.test.ts @@ -0,0 +1,235 @@ +import { createHttpGatewayTarget, getHttpGateway, listHttpGatewayTargets } from '../agentcore-http-gateways.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +vi.mock('../account', () => ({ + getCredentialProvider: vi.fn().mockReturnValue({ + accessKeyId: 'AKID', + secretAccessKey: 'SECRET', + sessionToken: 'TOKEN', + }), +})); + +vi.mock('@smithy/signature-v4', () => ({ + SignatureV4: class { + // eslint-disable-next-line @typescript-eslint/require-await + async sign(request: { headers: Record }) { + return { headers: { ...request.headers, Authorization: 'signed' } }; + } + }, +})); + +vi.mock('@aws-crypto/sha256-js', () => ({ + Sha256: class {}, +})); + +vi.mock('@aws-sdk/credential-provider-node', () => ({ + defaultProvider: vi.fn(), +})); + +function mockJsonResponse(body: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + }; +} + +describe('agentcore-http-gateways', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('createHttpGatewayTarget', () => { + it('sends agentcoreRuntime in request body', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + targetId: 'tgt-001', + name: 'my-target', + status: 'CREATING', + }) + ); + + const result = await createHttpGatewayTarget({ + region: 'us-east-1', + gatewayId: 'gw-123', + targetName: 'my-target', + runtimeArn: 'arn:aws:bedrock-agentcore:us-east-1:123:runtime/rt-1', + qualifier: 'DEFAULT', + }); + + expect(result.targetId).toBe('tgt-001'); + expect(result.name).toBe('my-target'); + expect(mockFetch).toHaveBeenCalledTimes(1); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.name).toBe('my-target'); + expect(body.targetConfiguration.http.agentcoreRuntime).toEqual({ + arn: 'arn:aws:bedrock-agentcore:us-east-1:123:runtime/rt-1', + qualifier: 'DEFAULT', + }); + expect(body.credentialProviderConfigurations).toEqual([{ credentialProviderType: 'GATEWAY_IAM_ROLE' }]); + expect(body.clientToken).toBeDefined(); + }); + + it('falls back to runtimeTargetConfiguration on ValidationException', async () => { + // First call fails with ValidationException + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 400, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('ValidationException: Unknown field agentcoreRuntime'), + }); + // Second call (fallback) succeeds + mockFetch.mockResolvedValueOnce( + mockJsonResponse({ + targetId: 'tgt-002', + name: 'my-target', + status: 'CREATING', + }) + ); + + const result = await createHttpGatewayTarget({ + region: 'us-east-1', + gatewayId: 'gw-123', + targetName: 'my-target', + runtimeArn: 'arn:aws:bedrock-agentcore:us-east-1:123:runtime/rt-1', + }); + + expect(result.targetId).toBe('tgt-002'); + expect(mockFetch).toHaveBeenCalledTimes(2); + + // Second call should use runtimeTargetConfiguration + const fallbackBody = JSON.parse(mockFetch.mock.calls[1]![1].body); + expect(fallbackBody.targetConfiguration.http.runtimeTargetConfiguration).toEqual({ + arn: 'arn:aws:bedrock-agentcore:us-east-1:123:runtime/rt-1', + qualifier: 'DEFAULT', + }); + }); + + it('falls back to runtimeTargetConfiguration on 400 status', async () => { + // First call fails with 400 + mockFetch.mockResolvedValueOnce({ + ok: false, + status: 400, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('400 Bad Request'), + }); + // Second call (fallback) succeeds + mockFetch.mockResolvedValueOnce( + mockJsonResponse({ + targetId: 'tgt-003', + name: 'my-target', + status: 'CREATING', + }) + ); + + const result = await createHttpGatewayTarget({ + region: 'us-east-1', + gatewayId: 'gw-123', + targetName: 'my-target', + runtimeArn: 'arn:runtime', + }); + + expect(result.targetId).toBe('tgt-003'); + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('throws on non-validation errors (no fallback)', async () => { + mockFetch.mockResolvedValue({ + ok: false, + status: 500, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('Internal Server Error'), + }); + + await expect( + createHttpGatewayTarget({ + region: 'us-east-1', + gatewayId: 'gw-123', + targetName: 'my-target', + runtimeArn: 'arn:runtime', + }) + ).rejects.toThrow('Failed to create target'); + + // Only one call — no fallback attempt + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + }); + + describe('getHttpGateway', () => { + it('returns gateway details', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + gatewayId: 'gw-123', + gatewayArn: 'arn:aws:bedrock-agentcore:us-east-1:123:gateway/gw-123', + gatewayUrl: 'https://gw-123.example.com', + name: 'my-gateway', + status: 'READY', + authorizerType: 'AWS_IAM', + roleArn: 'arn:aws:iam::123:role/GwRole', + createdAt: '2026-01-01T00:00:00Z', + updatedAt: '2026-01-02T00:00:00Z', + }) + ); + + const result = await getHttpGateway({ region: 'us-east-1', gatewayId: 'gw-123' }); + + expect(result.gatewayId).toBe('gw-123'); + expect(result.name).toBe('my-gateway'); + expect(result.status).toBe('READY'); + expect(result.gatewayUrl).toBe('https://gw-123.example.com'); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/gateways/gw-123'), + expect.objectContaining({ method: 'GET' }) + ); + }); + }); + + describe('listHttpGatewayTargets', () => { + it('returns targets array', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + targets: [ + { targetId: 'tgt-1', name: 'target-1', status: 'READY' }, + { targetId: 'tgt-2', name: 'target-2', status: 'CREATING' }, + ], + }) + ); + + const result = await listHttpGatewayTargets({ + region: 'us-east-1', + gatewayId: 'gw-123', + }); + + expect(result.targets).toHaveLength(2); + expect(result.targets[0]!.targetId).toBe('tgt-1'); + expect(result.targets[0]!.name).toBe('target-1'); + expect(result.targets[1]!.targetId).toBe('tgt-2'); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/gateways/gw-123/targets'), + expect.objectContaining({ method: 'GET' }) + ); + }); + + it('handles response with items field instead of targets', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + items: [{ targetId: 'tgt-1', name: 'target-1', status: 'READY' }], + }) + ); + + const result = await listHttpGatewayTargets({ + region: 'us-east-1', + gatewayId: 'gw-123', + }); + + expect(result.targets).toHaveLength(1); + expect(result.targets[0]!.targetId).toBe('tgt-1'); + }); + }); +}); diff --git a/src/cli/aws/__tests__/agentcore-recommendation.test.ts b/src/cli/aws/__tests__/agentcore-recommendation.test.ts new file mode 100644 index 000000000..1b330cf30 --- /dev/null +++ b/src/cli/aws/__tests__/agentcore-recommendation.test.ts @@ -0,0 +1,295 @@ +import { + deleteRecommendation, + getRecommendation, + listRecommendations, + startRecommendation, +} from '../agentcore-recommendation.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +vi.mock('../account', () => ({ + getCredentialProvider: vi.fn().mockReturnValue({ + accessKeyId: 'AKID', + secretAccessKey: 'SECRET', + sessionToken: 'TOKEN', + }), +})); + +vi.mock('@smithy/signature-v4', () => ({ + SignatureV4: class { + // eslint-disable-next-line @typescript-eslint/require-await + async sign(request: { headers: Record }) { + return { headers: { ...request.headers, Authorization: 'signed' } }; + } + }, +})); + +vi.mock('@aws-crypto/sha256-js', () => ({ + Sha256: class {}, +})); + +vi.mock('@aws-sdk/credential-provider-node', () => ({ + defaultProvider: vi.fn(), +})); + +function mockJsonResponse(body: unknown, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + json: () => Promise.resolve(body), + text: () => Promise.resolve(JSON.stringify(body)), + }; +} + +describe('agentcore-recommendation', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe('startRecommendation', () => { + it('sends POST to /recommendations with correct body', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + recommendationId: 'rec-123', + recommendationArn: 'arn:rec-123', + name: 'MyRecommendation', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + status: 'PENDING', + }) + ); + + const result = await startRecommendation({ + region: 'us-west-2', + name: 'MyRecommendation', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + recommendationConfig: { + systemPromptRecommendationConfig: { + systemPrompt: { text: 'You are a helpful agent.' }, + agentTraces: { + cloudwatchLogs: { + logGroupArns: ['arn:log-group'], + serviceNames: ['bedrock-agentcore'], + startTime: '2026-03-23T00:00:00.000Z', + endTime: '2026-03-30T00:00:00.000Z', + }, + }, + evaluationConfig: { + evaluators: [{ evaluatorArn: 'arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness' }], + }, + }, + }, + }); + + expect(result.recommendationId).toBe('rec-123'); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/recommendations'), + expect.objectContaining({ method: 'POST' }) + ); + + const fetchCall = mockFetch.mock.calls[0]!; + const body = JSON.parse(fetchCall[1].body); + expect(body.name).toBe('MyRecommendation'); + expect(body.type).toBe('SYSTEM_PROMPT_RECOMMENDATION'); + expect(body.recommendationConfig.systemPromptRecommendationConfig).toBeDefined(); + }); + + it('omits description when not provided', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + recommendationId: 'r1', + recommendationArn: 'arn:1', + name: 'MyRec', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + status: 'PENDING', + }) + ); + + await startRecommendation({ + region: 'us-west-2', + name: 'MyRec', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + recommendationConfig: { + systemPromptRecommendationConfig: { + systemPrompt: { text: '' }, + agentTraces: { + cloudwatchLogs: { + logGroupArns: [], + serviceNames: ['bedrock-agentcore'], + startTime: '2026-03-23T00:00:00.000Z', + endTime: '2026-03-30T00:00:00.000Z', + }, + }, + evaluationConfig: { + evaluators: [{ evaluatorArn: 'arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness' }], + }, + }, + }, + }); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.description).toBeUndefined(); + }); + + it('includes description when provided', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + recommendationId: 'r1', + recommendationArn: 'arn:1', + name: 'MyRec', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + status: 'PENDING', + }) + ); + + await startRecommendation({ + region: 'us-west-2', + name: 'MyRec', + description: 'Test description', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + recommendationConfig: { + systemPromptRecommendationConfig: { + systemPrompt: { text: '' }, + agentTraces: { + cloudwatchLogs: { + logGroupArns: [], + serviceNames: ['bedrock-agentcore'], + startTime: '2026-03-23T00:00:00.000Z', + endTime: '2026-03-30T00:00:00.000Z', + }, + }, + evaluationConfig: { + evaluators: [{ evaluatorArn: 'arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness' }], + }, + }, + }, + }); + + const body = JSON.parse(mockFetch.mock.calls[0]![1].body); + expect(body.description).toBe('Test description'); + }); + + it('throws on non-ok response', async () => { + mockFetch.mockResolvedValue({ + ok: false, + status: 400, + headers: new Map([['x-amzn-requestid', 'test-request-id']]), + text: () => Promise.resolve('Bad Request'), + }); + + await expect( + startRecommendation({ + region: 'us-west-2', + name: 'MyRec', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + recommendationConfig: {}, + }) + ).rejects.toThrow('Recommendation API error (400)'); + }); + }); + + describe('getRecommendation', () => { + it('sends GET to /recommendations/{id}', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + recommendationId: 'rec-123', + recommendationArn: 'arn:rec-123', + name: 'MyRec', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + status: 'COMPLETED', + recommendationResult: { + systemPromptRecommendationResult: { + recommendedSystemPrompt: 'Optimized prompt', + explanation: 'Made it better', + }, + }, + }) + ); + + const result = await getRecommendation({ region: 'us-west-2', recommendationId: 'rec-123' }); + + expect(result.recommendationId).toBe('rec-123'); + expect(result.name).toBe('MyRec'); + expect(result.recommendationResult?.systemPromptRecommendationResult?.recommendedSystemPrompt).toBe( + 'Optimized prompt' + ); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/recommendations/rec-123'), + expect.objectContaining({ method: 'GET' }) + ); + }); + }); + + describe('deleteRecommendation', () => { + it('sends DELETE to /recommendations/{id}', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({ recommendationId: 'rec-123', status: 'DELETING' }, 200)); + + const result = await deleteRecommendation({ region: 'us-west-2', recommendationId: 'rec-123' }); + + expect(result.recommendationId).toBe('rec-123'); + expect(result.status).toBe('DELETING'); + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('/recommendations/rec-123'), + expect.objectContaining({ method: 'DELETE' }) + ); + }); + + it('throws on failure', async () => { + mockFetch.mockRejectedValue(new Error('Network error')); + + await expect(deleteRecommendation({ region: 'us-west-2', recommendationId: 'rec-123' })).rejects.toThrow( + 'Network error' + ); + }); + }); + + describe('listRecommendations', () => { + it('sends GET to /recommendations', async () => { + mockFetch.mockResolvedValue( + mockJsonResponse({ + recommendationSummaries: [ + { + recommendationId: 'r1', + recommendationArn: 'arn:r1', + name: 'Rec1', + type: 'SYSTEM_PROMPT_RECOMMENDATION', + status: 'COMPLETED', + }, + { + recommendationId: 'r2', + recommendationArn: 'arn:r2', + name: 'Rec2', + type: 'TOOL_DESCRIPTION_RECOMMENDATION', + status: 'COMPLETED', + }, + ], + }) + ); + + const result = await listRecommendations({ region: 'us-west-2' }); + + expect(result.recommendationSummaries).toHaveLength(2); + expect(result.recommendationSummaries[0]!.name).toBe('Rec1'); + }); + + it('passes maxResults and nextToken as query params', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({ recommendationSummaries: [] })); + + await listRecommendations({ region: 'us-west-2', maxResults: 10, nextToken: 'abc' }); + + const url = mockFetch.mock.calls[0]![0] as string; + expect(url).toContain('maxResults=10'); + expect(url).toContain('nextToken=abc'); + }); + + it('returns empty array when response has no recommendationSummaries', async () => { + mockFetch.mockResolvedValue(mockJsonResponse({})); + + const result = await listRecommendations({ region: 'us-west-2' }); + + expect(result.recommendationSummaries).toEqual([]); + }); + }); +}); diff --git a/src/cli/aws/__tests__/agentcore.test.ts b/src/cli/aws/__tests__/agentcore.test.ts index e26e4324e..f23ff5c83 100644 --- a/src/cli/aws/__tests__/agentcore.test.ts +++ b/src/cli/aws/__tests__/agentcore.test.ts @@ -1,4 +1,4 @@ -import { extractResult, parseA2AResponse, parseSSE, parseSSELine } from '../agentcore.js'; +import { buildBearerInvokeHeaders, extractResult, parseA2AResponse, parseSSE, parseSSELine } from '../agentcore.js'; import { describe, expect, it } from 'vitest'; describe('parseSSELine', () => { @@ -176,3 +176,43 @@ describe('parseA2AResponse', () => { expect(parseA2AResponse('not json')).toBe('not json'); }); }); + +describe('buildBearerInvokeHeaders', () => { + it('includes custom headers from options.headers', () => { + const headers = buildBearerInvokeHeaders( + { + bearerToken: 'tok', + headers: { + 'x-amzn-bedrock-agentcore-runtime-custom-foo': 'bar', + 'x-amzn-bedrock-agentcore-runtime-custom-baz': 'qux', + }, + }, + 'application/json' + ); + expect(headers['x-amzn-bedrock-agentcore-runtime-custom-foo']).toBe('bar'); + expect(headers['x-amzn-bedrock-agentcore-runtime-custom-baz']).toBe('qux'); + }); + + it('sets Authorization, Content-Type, Accept, and default user ID', () => { + const headers = buildBearerInvokeHeaders({ bearerToken: 'tok' }, 'application/json'); + expect(headers.Authorization).toBe('Bearer tok'); + expect(headers['Content-Type']).toBe('application/json'); + expect(headers.Accept).toBe('application/json'); + expect(headers['X-Amzn-Bedrock-AgentCore-Runtime-User-Id']).toBe('default-user'); + }); + + it('sets session ID header when provided', () => { + const headers = buildBearerInvokeHeaders({ bearerToken: 'tok', sessionId: 's1' }, 'application/json'); + expect(headers['X-Amzn-Bedrock-AgentCore-Runtime-Session-Id']).toBe('s1'); + }); + + it('omits session ID header when not provided', () => { + const headers = buildBearerInvokeHeaders({ bearerToken: 'tok' }, 'application/json'); + expect(headers).not.toHaveProperty('X-Amzn-Bedrock-AgentCore-Runtime-Session-Id'); + }); + + it('returns correct headers when options.headers is undefined', () => { + const headers = buildBearerInvokeHeaders({ bearerToken: 'tok' }, 'application/json'); + expect(Object.keys(headers)).toHaveLength(4); // Authorization, Content-Type, Accept, User-Id + }); +}); diff --git a/src/cli/aws/__tests__/api-client.test.ts b/src/cli/aws/__tests__/api-client.test.ts new file mode 100644 index 000000000..5ebc1ebd3 --- /dev/null +++ b/src/cli/aws/__tests__/api-client.test.ts @@ -0,0 +1,185 @@ +import { AgentCoreApiClient, AgentCoreApiError } from '../api-client.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { mockSign } = vi.hoisted(() => ({ + mockSign: vi.fn(), +})); + +vi.mock('../account', () => ({ + getCredentialProvider: vi.fn().mockReturnValue({}), +})); + +vi.mock('@smithy/signature-v4', () => ({ + SignatureV4: class { + sign = mockSign; + }, +})); + +vi.mock('@smithy/protocol-http', () => ({ + HttpRequest: class { + constructor(public opts: unknown) {} + }, +})); + +vi.mock('@aws-crypto/sha256-js', () => ({ + Sha256: class {}, +})); + +vi.mock('@aws-sdk/credential-provider-node', () => ({ + defaultProvider: vi.fn().mockReturnValue({}), +})); + +const mockFetch = vi.fn(); +vi.stubGlobal('fetch', mockFetch); + +describe('AgentCoreApiClient', () => { + beforeEach(() => { + vi.clearAllMocks(); + delete process.env.AGENTCORE_STAGE; + mockSign.mockResolvedValue({ headers: { host: 'example.com', 'content-type': 'application/json' } }); + }); + + describe('endpoint resolution', () => { + it('uses control plane prod endpoint by default', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 200 })); + + await client.request({ method: 'GET', path: '/test' }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('bedrock-agentcore-control.us-west-2.amazonaws.com'), + expect.anything() + ); + }); + + it('uses data plane prod endpoint', async () => { + const client = new AgentCoreApiClient({ region: 'us-east-1', plane: 'data' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + await client.request({ method: 'GET', path: '/test' }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('bedrock-agentcore.us-east-1.amazonaws.com'), + expect.anything() + ); + }); + + it('uses beta control plane endpoint when AGENTCORE_STAGE=beta', async () => { + process.env.AGENTCORE_STAGE = 'beta'; + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + await client.request({ method: 'GET', path: '/test' }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('beta.us-west-2.elcapcp.genesis-primitives.aws.dev'), + expect.anything() + ); + }); + + it('uses gamma data plane endpoint when AGENTCORE_STAGE=gamma', async () => { + process.env.AGENTCORE_STAGE = 'gamma'; + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'data' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + await client.request({ method: 'GET', path: '/test' }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.stringContaining('gamma.us-west-2.elcapdp.genesis-primitives.aws.dev'), + expect.anything() + ); + }); + }); + + describe('request()', () => { + it('returns parsed JSON on success', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({ harnessId: 'h-123' }), { status: 200 })); + + const result = await client.request({ method: 'GET', path: '/harnesses/h-123' }); + + expect(result).toEqual({ harnessId: 'h-123' }); + }); + + it('returns empty object on 204', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(null, { status: 204 })); + + const result = await client.request({ method: 'DELETE', path: '/harnesses/h-123' }); + + expect(result).toEqual({}); + }); + + it('throws AgentCoreApiError on non-2xx', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue( + new Response('{"message":"Not found"}', { + status: 404, + headers: { 'x-amzn-requestid': 'req-abc' }, + }) + ); + + const err = await client.request({ method: 'GET', path: '/harnesses/bad' }).catch((e: unknown) => e); + expect(err).toBeInstanceOf(AgentCoreApiError); + const apiErr = err as AgentCoreApiError; + expect(apiErr.statusCode).toBe(404); + expect(apiErr.requestId).toBe('req-abc'); + expect(apiErr.errorBody).toContain('Not found'); + }); + + it('sends JSON body when provided', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({ ok: true }), { status: 201 })); + + await client.request({ method: 'POST', path: '/harnesses', body: { harnessName: 'test' } }); + + expect(mockFetch).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ body: JSON.stringify({ harnessName: 'test' }) }) + ); + }); + + it('appends query parameters to URL', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'control' }); + mockFetch.mockResolvedValue(new Response(JSON.stringify({}), { status: 200 })); + + await client.request({ method: 'GET', path: '/harnesses', query: { maxResults: '10' } }); + + expect(mockFetch).toHaveBeenCalledWith(expect.stringContaining('maxResults=10'), expect.anything()); + }); + }); + + describe('requestRaw()', () => { + it('returns raw Response object', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'data' }); + const mockResponse = new Response('streaming data', { status: 200 }); + mockFetch.mockResolvedValue(mockResponse); + + const response = await client.requestRaw({ method: 'POST', path: '/harnesses/invoke' }); + + expect(response).toBe(mockResponse); + expect(response.status).toBe(200); + }); + + it('passes custom headers through', async () => { + const client = new AgentCoreApiClient({ region: 'us-west-2', plane: 'data' }); + mockFetch.mockResolvedValue(new Response('', { status: 200 })); + + await client.requestRaw({ + method: 'POST', + path: '/harnesses/invoke', + headers: { 'X-Amzn-Bedrock-AgentCore-Runtime-Session-Id': 'sess-123' }, + }); + + expect(mockSign).toHaveBeenCalledWith( + expect.objectContaining({ + opts: expect.objectContaining({ + headers: expect.objectContaining({ + 'X-Amzn-Bedrock-AgentCore-Runtime-Session-Id': 'sess-123', + }), + }), + }) + ); + }); + }); +}); diff --git a/src/cli/aws/__tests__/poll.test.ts b/src/cli/aws/__tests__/poll.test.ts new file mode 100644 index 000000000..2894758d8 --- /dev/null +++ b/src/cli/aws/__tests__/poll.test.ts @@ -0,0 +1,92 @@ +import { PollFailureError, PollTimeoutError, pollUntilTerminal } from '../poll.js'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +interface MockStatus { + status: string; + reason?: string; +} + +describe('pollUntilTerminal', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns immediately when first result is terminal', async () => { + const fn = vi.fn().mockResolvedValue({ status: 'READY' }); + + const result = await pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => r.status === 'READY', + }); + + expect(result).toEqual({ status: 'READY' }); + expect(fn).toHaveBeenCalledTimes(1); + }); + + it('polls until terminal status is reached', async () => { + const fn = vi + .fn() + .mockResolvedValueOnce({ status: 'CREATING' }) + .mockResolvedValueOnce({ status: 'CREATING' }) + .mockResolvedValueOnce({ status: 'READY' }); + + const result = await pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => ['READY', 'FAILED'].includes(r.status), + intervalMs: 10, + }); + + expect(result).toEqual({ status: 'READY' }); + expect(fn).toHaveBeenCalledTimes(3); + }); + + it('throws PollFailureError when failure state is detected', async () => { + const fn = vi.fn().mockResolvedValue({ status: 'FAILED', reason: 'bad config' }); + + await expect( + pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => ['READY', 'FAILED'].includes(r.status), + isFailure: (r: MockStatus) => r.status === 'FAILED', + getFailureReason: (r: MockStatus) => `Harness failed: ${r.reason}`, + intervalMs: 10, + }) + ).rejects.toThrow(PollFailureError); + + await expect( + pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => ['READY', 'FAILED'].includes(r.status), + isFailure: (r: MockStatus) => r.status === 'FAILED', + getFailureReason: (r: MockStatus) => `Harness failed: ${r.reason}`, + intervalMs: 10, + }) + ).rejects.toThrow('Harness failed: bad config'); + }); + + it('throws PollTimeoutError when maxWaitMs exceeded', async () => { + const fn = vi.fn().mockResolvedValue({ status: 'CREATING' }); + + await expect( + pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => r.status === 'READY', + intervalMs: 10, + maxWaitMs: 50, + }) + ).rejects.toThrow(PollTimeoutError); + }); + + it('uses default failure message when getFailureReason is not provided', async () => { + const fn = vi.fn().mockResolvedValue({ status: 'FAILED' }); + + await expect( + pollUntilTerminal({ + fn, + isTerminal: (r: MockStatus) => r.status === 'FAILED', + isFailure: (r: MockStatus) => r.status === 'FAILED', + intervalMs: 10, + }) + ).rejects.toThrow('Resource entered a failed state'); + }); +}); diff --git a/src/cli/aws/agentcore-ab-tests.ts b/src/cli/aws/agentcore-ab-tests.ts new file mode 100644 index 000000000..4bcf0ce16 --- /dev/null +++ b/src/cli/aws/agentcore-ab-tests.ts @@ -0,0 +1,360 @@ +/** + * AWS client wrappers for AB Test data plane operations. + * + * Uses the AgentCore Evaluation DataPlane API (bedrock-agentcore) + * with direct HTTP requests and SigV4 signing. + */ +import { getCredentialProvider } from './account'; +import { dnsSuffix } from './partition'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { HttpRequest } from '@smithy/protocol-http'; +import { SignatureV4 } from '@smithy/signature-v4'; +import { randomUUID } from 'node:crypto'; + +// ============================================================================ +// Types +// ============================================================================ + +export interface ABTestVariant { + name: 'C' | 'T1'; + weight: number; + variantConfiguration: { + configurationBundle?: { + bundleArn: string; + bundleVersion: string; + }; + target?: { + name: string; + }; + }; +} + +export type ABTestEvaluationConfig = + | { onlineEvaluationConfigArn: string } + | { + perVariantOnlineEvaluationConfig: { + name: 'C' | 'T1'; + onlineEvaluationConfigArn: string; + }[]; + }; + +export interface GatewayFilter { + targetPaths: string[]; +} + +export interface TrafficAllocationConfig { + routeOnHeader: { + headerName: string; + }; +} + +export interface ConfidenceInterval { + lower?: number; + upper?: number; +} + +export interface ControlStats { + treatmentName: string; + sampleSize: number; + mean: number; +} + +export interface VariantResult { + treatmentName: string; + sampleSize: number; + mean: number; + absoluteChange?: number; + percentChange?: number; + pValue?: number; + confidenceInterval?: ConfidenceInterval; + isSignificant: boolean; +} + +export interface EvaluatorMetric { + evaluatorArn: string; + controlStats: ControlStats; + variantResults: VariantResult[]; +} + +export interface ABTestResults { + analysisTimestamp?: string; + evaluatorMetrics: EvaluatorMetric[]; +} + +// ── Create ────────────────────────────────────────────────────────────────── + +export interface CreateABTestOptions { + region: string; + name: string; + description?: string; + gatewayArn: string; + roleArn: string; + variants: ABTestVariant[]; + evaluationConfig: ABTestEvaluationConfig; + gatewayFilter?: GatewayFilter; + trafficAllocationConfig?: TrafficAllocationConfig; + maxDurationDays?: number; + enableOnCreate?: boolean; +} + +export interface CreateABTestResult { + abTestId: string; + abTestArn: string; + name?: string; + status: string; + executionStatus: string; + createdAt: string; +} + +// ── Get ───────────────────────────────────────────────────────────────────── + +export interface GetABTestOptions { + region: string; + abTestId: string; +} + +export interface GetABTestResult { + abTestId: string; + abTestArn: string; + name: string; + description?: string; + status: string; + executionStatus: string; + gatewayArn: string; + roleArn: string; + variants: ABTestVariant[]; + evaluationConfig: ABTestEvaluationConfig; + trafficAllocationConfig?: TrafficAllocationConfig; + maxDurationDays?: number; + currentRunId?: string; + stopReason?: string; + failureReason?: string; + startedAt?: string; + stoppedAt?: string; + maxDurationExpiresAt?: string; + createdAt: string; + updatedAt: string; + results?: ABTestResults; +} + +// ── Update ────────────────────────────────────────────────────────────────── + +export interface UpdateABTestOptions { + region: string; + abTestId: string; + name?: string; + description?: string; + variants?: ABTestVariant[]; + trafficAllocationConfig?: TrafficAllocationConfig; + evaluationConfig?: ABTestEvaluationConfig; + maxDurationDays?: number; + executionStatus?: 'PAUSED' | 'RUNNING' | 'STOPPED'; + roleArn?: string; +} + +export interface UpdateABTestResult { + abTestId: string; + abTestArn: string; + status: string; + executionStatus: string; + failureReason?: string; + updatedAt: string; +} + +// ── Delete ────────────────────────────────────────────────────────────────── + +export interface DeleteABTestOptions { + region: string; + abTestId: string; +} + +// ── List ──────────────────────────────────────────────────────────────────── + +export interface ListABTestsOptions { + region: string; + maxResults?: number; + nextToken?: string; +} + +export interface ABTestSummary { + abTestId: string; + abTestArn: string; + name: string; + description?: string; + status: string; + executionStatus: string; + gatewayArn?: string; + createdAt: string; + updatedAt: string; +} + +export interface ListABTestsResult { + abTests: ABTestSummary[]; + nextToken?: string; +} + +// ============================================================================ +// HTTP signing helpers +// ============================================================================ + +function getDataPlaneEndpoint(region: string): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapdp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapdp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore.${region}.${dnsSuffix(region)}`; +} + +async function signedRequestToEndpoint( + endpoint: string, + options: { + region: string; + method: string; + path: string; + body?: string; + } +): Promise { + const { region, method, path, body } = options; + const url = new URL(path, endpoint); + + const query: Record = {}; + url.searchParams.forEach((value, key) => { + query[key] = value; + }); + + const request = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname, + ...(Object.keys(query).length > 0 && { query }), + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + }, + ...(body && { body }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const service = 'bedrock-agentcore'; + const signer = new SignatureV4({ + service, + region, + credentials, + sha256: Sha256, + }); + + const signedReq = await signer.sign(request); + + const response = await fetch(`${endpoint}${path}`, { + method, + headers: signedReq.headers as Record, + ...(body && { body }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error(`ABTest API error (${response.status}): ${errorBody}`); + } + + if (response.status === 204) return {}; + return response.json(); +} + +/** Data plane request — used for GetABTest (includes results/metrics). */ +async function dpRequest(options: { region: string; method: string; path: string; body?: string }): Promise { + return signedRequestToEndpoint(getDataPlaneEndpoint(options.region), options); +} + +// ============================================================================ +// Control Plane Operations (CRUD) +// ============================================================================ + +export async function createABTest(options: CreateABTestOptions): Promise { + const body = JSON.stringify({ + name: options.name, + clientToken: randomUUID(), + gatewayArn: options.gatewayArn, + roleArn: options.roleArn, + variants: options.variants, + evaluationConfig: options.evaluationConfig, + ...(options.description && { description: options.description }), + ...(options.gatewayFilter && { gatewayFilter: options.gatewayFilter }), + ...(options.trafficAllocationConfig && { trafficAllocationConfig: options.trafficAllocationConfig }), + ...(options.maxDurationDays !== undefined && { maxDurationDays: options.maxDurationDays }), + ...(options.enableOnCreate !== undefined && { enableOnCreate: options.enableOnCreate }), + }); + + const result = await dpRequest({ + region: options.region, + method: 'POST', + path: '/ab-tests', + body, + }); + + return result as CreateABTestResult; +} + +export async function getABTest(options: GetABTestOptions): Promise { + // Data plane includes results/metrics in the response + const data = await dpRequest({ + region: options.region, + method: 'GET', + path: `/ab-tests/${options.abTestId}`, + }); + + return data as GetABTestResult; +} + +export async function updateABTest(options: UpdateABTestOptions): Promise { + const body: Record = { clientToken: randomUUID() }; + if (options.name !== undefined) body.name = options.name; + if (options.description !== undefined) body.description = options.description; + if (options.variants !== undefined) body.variants = options.variants; + if (options.trafficAllocationConfig !== undefined) body.trafficAllocationConfig = options.trafficAllocationConfig; + if (options.evaluationConfig !== undefined) body.evaluationConfig = options.evaluationConfig; + if (options.maxDurationDays !== undefined) body.maxDurationDays = options.maxDurationDays; + if (options.executionStatus !== undefined) body.executionStatus = options.executionStatus; + if (options.roleArn !== undefined) body.roleArn = options.roleArn; + + const data = await dpRequest({ + region: options.region, + method: 'PUT', + path: `/ab-tests/${options.abTestId}`, + body: JSON.stringify(body), + }); + + return data as UpdateABTestResult; +} + +export async function deleteABTest(options: DeleteABTestOptions): Promise<{ success: boolean; error?: string }> { + try { + await dpRequest({ + region: options.region, + method: 'DELETE', + path: `/ab-tests/${options.abTestId}`, + }); + return { success: true }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +export async function listABTests(options: ListABTestsOptions): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + const query = params.toString(); + + const data = await dpRequest({ + region: options.region, + method: 'GET', + path: `/ab-tests${query ? `?${query}` : ''}`, + }); + + const result = data as ListABTestsResult; + return { + abTests: result.abTests ?? [], + nextToken: result.nextToken, + }; +} diff --git a/src/cli/aws/agentcore-batch-evaluation.ts b/src/cli/aws/agentcore-batch-evaluation.ts new file mode 100644 index 000000000..9b0923753 --- /dev/null +++ b/src/cli/aws/agentcore-batch-evaluation.ts @@ -0,0 +1,411 @@ +/** + * AWS client wrappers for BatchEvaluation operations. + * + * The BatchEvaluation API is a flat, stateless model — no persistent "job" resource. + * Each batch evaluation is started, polled, and optionally stopped. + * + * Endpoints: + * POST /evaluations/batch-evaluate → StartBatchEvaluation + * GET /evaluations/batch-evaluate/{batchEvaluationId} → GetBatchEvaluation + * GET /evaluations/batch-evaluate → ListBatchEvaluations + * POST /evaluations/batch-evaluate/{batchEvaluationId}/stop → StopBatchEvaluation + * DELETE /evaluations/batch-evaluate/{batchEvaluationId} → DeleteBatchEvaluation + * + * Uses direct HTTP requests with SigV4 signing (service: bedrock-agentcore). + */ +import { getCredentialProvider } from './account'; +import { dnsSuffix } from './partition'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { HttpRequest } from '@smithy/protocol-http'; +import { SignatureV4 } from '@smithy/signature-v4'; + +// ============================================================================ +// Types +// ============================================================================ + +export interface SessionFilterConfig { + startTime?: string; + endTime?: string; +} + +export interface CloudWatchFilterConfig { + sessionIds?: string[]; + timeRange?: SessionFilterConfig; +} + +export interface CloudWatchLogsSource { + serviceNames: string[]; + logGroupNames: string[]; + filterConfig?: CloudWatchFilterConfig; +} + +export interface DataSourceConfig { + cloudWatchLogs?: CloudWatchLogsSource; + onlineEvaluationConfigSource?: Record; +} + +export interface Evaluator { + evaluatorId: string; +} + +export interface GroundTruthAssertion { + text: string; +} + +export interface GroundTruthTurnInput { + prompt: string; +} + +export interface GroundTruthTurnExpectedResponse { + text: string; +} + +export interface GroundTruthTurn { + input: GroundTruthTurnInput; + expectedResponse: GroundTruthTurnExpectedResponse; +} + +export interface ExpectedTrajectory { + toolNames: string[]; +} + +export interface InlineGroundTruth { + assertions?: GroundTruthAssertion[]; + expectedTrajectory?: ExpectedTrajectory; + turns?: GroundTruthTurn[]; +} + +export interface GroundTruth { + inline: InlineGroundTruth; +} + +export interface SessionMetadataEntry { + sessionId: string; + testScenarioId?: string; + groundTruth?: GroundTruth; + metadata?: Record; +} + +export interface EvaluationMetadata { + sessionMetadata?: SessionMetadataEntry[]; +} + +export interface StartBatchEvaluationOptions { + region: string; + name: string; + evaluators: Evaluator[]; + dataSourceConfig: DataSourceConfig; + evaluationMetadata?: EvaluationMetadata; + description?: string; + clientToken?: string; +} + +export interface StartBatchEvaluationResult { + batchEvaluationId: string; + batchEvaluationArn: string; + name: string; + status: string; + createdAt?: string; +} + +export interface GetBatchEvaluationOptions { + region: string; + batchEvaluationId: string; +} + +export interface CloudWatchOutputConfig { + logGroupName: string; + logStreamName: string; +} + +export interface OutputConfig { + cloudWatchConfig?: CloudWatchOutputConfig; +} + +export interface EvaluatorSummary { + evaluatorId: string; + statistics?: { + averageScore?: number; + averageTokenUsage?: { + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + }; + }; + totalEvaluated?: number; + totalFailed?: number; +} + +export interface EvaluationResults { + evaluatorSummaries?: EvaluatorSummary[]; + numberOfSessionsCompleted?: number; + numberOfSessionsFailed?: number; + numberOfSessionsInProgress?: number; + totalNumberOfSessions?: number; + numberOfSessionsIgnored?: number; +} + +export interface GetBatchEvaluationResult { + batchEvaluationId: string; + batchEvaluationArn: string; + name: string; + status: string; + createdAt?: string; + updatedAt?: string; + evaluators?: Evaluator[]; + dataSourceConfig?: DataSourceConfig; + outputConfig?: OutputConfig; + evaluationResults?: EvaluationResults; + errorDetails?: string[]; + description?: string; +} + +export interface BatchEvaluationResultEntry { + evaluatorId: string; + score?: number; + label?: string; + explanation?: string; + error?: string; +} + +export interface ListBatchEvaluationsOptions { + region: string; + maxResults?: number; + nextToken?: string; +} + +export interface BatchEvaluationSummary { + batchEvaluationId: string; + batchEvaluationArn: string; + name: string; + status: string; + createdAt?: string; + description?: string; + evaluators?: Evaluator[]; + evaluationResults?: EvaluationResults; + errorDetails?: string[]; +} + +export interface ListBatchEvaluationsResult { + batchEvaluations: BatchEvaluationSummary[]; + nextToken?: string; +} + +export interface StopBatchEvaluationOptions { + region: string; + batchEvaluationId: string; +} + +export interface StopBatchEvaluationResult { + batchEvaluationId: string; + batchEvaluationArn: string; + status: string; + description?: string; +} + +export interface DeleteBatchEvaluationOptions { + region: string; + batchEvaluationId: string; +} + +export interface DeleteBatchEvaluationResult { + batchEvaluationId: string; + batchEvaluationArn: string; + status: string; +} + +// ============================================================================ +// HTTP signing helper +// ============================================================================ + +function getEndpoint(region: string): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapdp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapdp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore.${region}.${dnsSuffix(region)}`; +} + +async function signedRequest(options: { + region: string; + method: string; + path: string; + body?: string; +}): Promise<{ data: unknown; status: number }> { + const { region, method, path, body } = options; + const endpoint = getEndpoint(region); + const url = new URL(path, endpoint); + + const request = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname + url.search, + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + }, + ...(body && { body }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const signer = new SignatureV4({ + service: 'bedrock-agentcore', + region, + credentials, + sha256: Sha256, + }); + + const signedReq = await signer.sign(request); + + const response = await fetch(`${endpoint}${url.pathname}${url.search}`, { + method, + headers: signedReq.headers as Record, + ...(body && { body }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error(`BatchEvaluation API error (${response.status}): ${errorBody}`); + } + + if (response.status === 204) return { data: {}, status: 204 }; + return { data: await response.json(), status: response.status }; +} + +// ============================================================================ +// API Operations +// ============================================================================ + +/** + * Start a batch evaluation (async — returns immediately with an ID to poll). + */ +export async function startBatchEvaluation(options: StartBatchEvaluationOptions): Promise { + const body: Record = { + batchEvaluationName: options.name, + evaluators: options.evaluators, + dataSourceConfig: options.dataSourceConfig, + }; + if (options.evaluationMetadata) { + body.evaluationMetadata = options.evaluationMetadata; + } + if (options.description) { + body.description = options.description; + } + if (options.clientToken) { + body.clientToken = options.clientToken; + } + + const { data } = await signedRequest({ + region: options.region, + method: 'POST', + path: '/evaluations/batch-evaluate', + body: JSON.stringify(body), + }); + + const raw = data as Record; + return { + batchEvaluationId: (raw.batchEvaluationId ?? '') as string, + batchEvaluationArn: (raw.batchEvaluationArn ?? '') as string, + name: (raw.batchEvaluationName ?? '') as string, + status: (raw.status ?? '') as string, + createdAt: raw.createdAt as string | undefined, + }; +} + +/** + * Get status and results of a batch evaluation. + */ +export async function getBatchEvaluation(options: GetBatchEvaluationOptions): Promise { + const { data } = await signedRequest({ + region: options.region, + method: 'GET', + path: `/evaluations/batch-evaluate/${options.batchEvaluationId}`, + }); + + const raw = data as Record; + return { + batchEvaluationId: (raw.batchEvaluationId ?? '') as string, + batchEvaluationArn: (raw.batchEvaluationArn ?? '') as string, + name: (raw.batchEvaluationName ?? '') as string, + status: (raw.status ?? '') as string, + createdAt: raw.createdAt as string | undefined, + updatedAt: raw.updatedAt as string | undefined, + evaluators: raw.evaluators as Evaluator[] | undefined, + dataSourceConfig: raw.dataSourceConfig as DataSourceConfig | undefined, + outputConfig: raw.outputConfig as OutputConfig | undefined, + evaluationResults: raw.evaluationResults as EvaluationResults | undefined, + errorDetails: raw.errorDetails as string[] | undefined, + description: raw.description as string | undefined, + }; +} + +/** + * List batch evaluations. + */ +export async function listBatchEvaluations(options: ListBatchEvaluationsOptions): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + + const query = params.toString(); + const path = `/evaluations/batch-evaluate${query ? `?${query}` : ''}`; + + const { data } = await signedRequest({ + region: options.region, + method: 'GET', + path, + }); + + const result = data as ListBatchEvaluationsResult; + return { + batchEvaluations: result.batchEvaluations ?? [], + nextToken: result.nextToken, + }; +} + +/** + * Stop a running batch evaluation. + */ +export async function stopBatchEvaluation(options: StopBatchEvaluationOptions): Promise { + const { data } = await signedRequest({ + region: options.region, + method: 'POST', + path: `/evaluations/batch-evaluate/${options.batchEvaluationId}/stop`, + }); + + const raw = data as Record; + return { + batchEvaluationId: (raw.batchEvaluationId ?? '') as string, + batchEvaluationArn: (raw.batchEvaluationArn ?? '') as string, + status: (raw.status ?? '') as string, + description: raw.description as string | undefined, + }; +} + +/** + * Delete a batch evaluation. + */ +export async function deleteBatchEvaluation( + options: DeleteBatchEvaluationOptions +): Promise { + const { data } = await signedRequest({ + region: options.region, + method: 'DELETE', + path: `/evaluations/batch-evaluate/${options.batchEvaluationId}`, + }); + + const raw = data as Record; + return { + batchEvaluationId: (raw.batchEvaluationId ?? '') as string, + batchEvaluationArn: (raw.batchEvaluationArn ?? '') as string, + status: (raw.status ?? '') as string, + }; +} + +/** + * Generate a client token for idempotency. + */ +export function generateClientToken(): string { + return crypto.randomUUID(); +} diff --git a/src/cli/aws/agentcore-config-bundles.ts b/src/cli/aws/agentcore-config-bundles.ts new file mode 100644 index 000000000..d890d95df --- /dev/null +++ b/src/cli/aws/agentcore-config-bundles.ts @@ -0,0 +1,368 @@ +/** + * AWS client wrappers for Configuration Bundle control plane operations. + * + * NOTE: The ConfigurationBundle API is not yet available in the + * @aws-sdk/client-bedrock-agentcore-control SDK. These wrappers use + * direct HTTP requests with SigV4 signing as an interim solution. + * When the SDK adds ConfigurationBundle commands, migrate to the SDK client. + */ +import { getCredentialProvider } from './account'; +import { dnsSuffix } from './partition'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { HttpRequest } from '@smithy/protocol-http'; +import { SignatureV4 } from '@smithy/signature-v4'; +import { randomUUID } from 'node:crypto'; + +// ============================================================================ +// Types +// ============================================================================ + +/** Freeform configuration for a component within a bundle. */ +export interface ComponentConfiguration { + configuration: Record; +} + +/** Map of component identifier (ARN) to its configuration. */ +export type ComponentConfigurationMap = Record; + +/** Version lineage metadata for git-like versioning. */ +export interface VersionLineageMetadata { + parentVersionIds?: string[]; + branchName?: string; + createdBy?: { name: string; arn?: string }; + commitMessage?: string; +} + +// ── Create ────────────────────────────────────────────────────────────────── + +export interface CreateConfigurationBundleOptions { + region: string; + bundleName: string; + description?: string; + components: ComponentConfigurationMap; + branchName?: string; + commitMessage?: string; + createdBy?: { name: string; arn?: string }; +} + +export interface CreateConfigurationBundleResult { + bundleArn: string; + bundleId: string; + versionId: string; + createdAt: string; +} + +// ── Get ───────────────────────────────────────────────────────────────────── + +export interface GetConfigurationBundleOptions { + region: string; + bundleId: string; + branchName?: string; +} + +export interface GetConfigurationBundleResult { + bundleArn: string; + bundleId: string; + bundleName: string; + description?: string; + versionId: string; + components: ComponentConfigurationMap; + lineageMetadata?: VersionLineageMetadata; + createdAt: string; + updatedAt: string; +} + +// ── Update ────────────────────────────────────────────────────────────────── + +export interface UpdateConfigurationBundleOptions { + region: string; + bundleId: string; + bundleName?: string; + description?: string; + components?: ComponentConfigurationMap; + parentVersionIds?: string[]; + branchName?: string; + commitMessage?: string; + createdBy?: { name: string; arn?: string }; +} + +export interface UpdateConfigurationBundleResult { + bundleArn: string; + bundleId: string; + versionId: string; + updatedAt: string; +} + +// ── Delete ────────────────────────────────────────────────────────────────── + +export interface DeleteConfigurationBundleOptions { + region: string; + bundleId: string; +} + +// ── List ──────────────────────────────────────────────────────────────────── + +export interface ListConfigurationBundlesOptions { + region: string; + maxResults?: number; + nextToken?: string; +} + +export interface ConfigurationBundleSummary { + bundleArn: string; + bundleId: string; + bundleName: string; + description?: string; +} + +export interface ListConfigurationBundlesResult { + bundles: ConfigurationBundleSummary[]; + nextToken?: string; +} + +// ── Get Version ───────────────────────────────────────────────────────────── + +export interface GetConfigurationBundleVersionOptions { + region: string; + bundleId: string; + versionId: string; +} + +export interface GetConfigurationBundleVersionResult { + bundleArn: string; + bundleId: string; + bundleName: string; + description?: string; + versionId: string; + components: ComponentConfigurationMap; + lineageMetadata?: VersionLineageMetadata; + createdAt: string; + versionCreatedAt: string; +} + +// ── List Versions ─────────────────────────────────────────────────────────── + +export interface ListConfigurationBundleVersionsFilter { + branchName?: string; + latestPerBranch?: boolean; + createdByName?: string; +} + +export interface ListConfigurationBundleVersionsOptions { + region: string; + bundleId: string; + maxResults?: number; + nextToken?: string; + filter?: ListConfigurationBundleVersionsFilter; +} + +export interface ConfigurationBundleVersionSummary { + bundleArn: string; + bundleId: string; + versionId: string; + lineageMetadata?: VersionLineageMetadata; + versionCreatedAt: string; +} + +export interface ListConfigurationBundleVersionsResult { + versions: ConfigurationBundleVersionSummary[]; + nextToken?: string; +} + +// ============================================================================ +// HTTP signing helper +// ============================================================================ + +// TODO: Remove beta/gamma endpoints before GA merge +function getControlPlaneEndpoint(region: string): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapcp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapcp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore-control.${region}.${dnsSuffix(region)}`; +} + +async function signedRequest(options: { + region: string; + method: string; + path: string; + body?: string; +}): Promise { + const { region, method, path, body } = options; + const endpoint = getControlPlaneEndpoint(region); + const url = new URL(path, endpoint); + + const query: Record = {}; + url.searchParams.forEach((value, key) => { + query[key] = value; + }); + + const request = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname, + ...(Object.keys(query).length > 0 && { query }), + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + }, + ...(body && { body }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const service = 'bedrock-agentcore'; + const signer = new SignatureV4({ + service, + region, + credentials, + sha256: Sha256, + }); + + const signedReq = await signer.sign(request); + + const response = await fetch(`${endpoint}${path}`, { + method, + headers: signedReq.headers as Record, + ...(body && { body }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error(`ConfigurationBundle API error (${response.status}): ${errorBody}`); + } + + if (response.status === 204) return {}; + return response.json(); +} + +// ============================================================================ +// Control Plane Operations +// ============================================================================ + +export async function createConfigurationBundle( + options: CreateConfigurationBundleOptions +): Promise { + const body = JSON.stringify({ + bundleName: options.bundleName, + clientToken: randomUUID(), + ...(options.description && { description: options.description }), + components: options.components, + ...(options.branchName && { branchName: options.branchName }), + ...(options.commitMessage && { commitMessage: options.commitMessage }), + ...(options.createdBy && { createdBy: options.createdBy }), + }); + + const result = await signedRequest({ + region: options.region, + method: 'POST', + path: '/configuration-bundles/create', + body, + }); + + return result as CreateConfigurationBundleResult; +} + +export async function getConfigurationBundle( + options: GetConfigurationBundleOptions +): Promise { + const params = new URLSearchParams(); + if (options.branchName) params.set('branchName', options.branchName); + const query = params.toString(); + const path = `/configuration-bundles/${options.bundleId}${query ? `?${query}` : ''}`; + + const data = await signedRequest({ + region: options.region, + method: 'GET', + path, + }); + + return data as GetConfigurationBundleResult; +} + +export async function updateConfigurationBundle( + options: UpdateConfigurationBundleOptions +): Promise { + const body: Record = { clientToken: randomUUID() }; + if (options.bundleName !== undefined) body.bundleName = options.bundleName; + if (options.description !== undefined) body.description = options.description; + if (options.components !== undefined) body.components = options.components; + if (options.parentVersionIds !== undefined) body.parentVersionIds = options.parentVersionIds; + if (options.branchName !== undefined) body.branchName = options.branchName; + if (options.commitMessage !== undefined) body.commitMessage = options.commitMessage; + if (options.createdBy !== undefined) body.createdBy = options.createdBy; + + const data = await signedRequest({ + region: options.region, + method: 'PUT', + path: `/configuration-bundles/${options.bundleId}`, + body: JSON.stringify(body), + }); + + return data as UpdateConfigurationBundleResult; +} + +export async function deleteConfigurationBundle(options: DeleteConfigurationBundleOptions): Promise { + await signedRequest({ + region: options.region, + method: 'DELETE', + path: `/configuration-bundles/${options.bundleId}`, + }); +} + +export async function listConfigurationBundles( + options: ListConfigurationBundlesOptions +): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + const query = params.toString(); + + const data = await signedRequest({ + region: options.region, + method: 'POST', + path: `/configuration-bundles${query ? `?${query}` : ''}`, + }); + + const result = data as ListConfigurationBundlesResult; + return { + bundles: result.bundles ?? [], + nextToken: result.nextToken, + }; +} + +export async function getConfigurationBundleVersion( + options: GetConfigurationBundleVersionOptions +): Promise { + const data = await signedRequest({ + region: options.region, + method: 'GET', + path: `/configuration-bundles/${options.bundleId}/versions/${options.versionId}`, + }); + + return data as GetConfigurationBundleVersionResult; +} + +export async function listConfigurationBundleVersions( + options: ListConfigurationBundleVersionsOptions +): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + const query = params.toString(); + + const body = options.filter ? JSON.stringify({ filter: options.filter }) : undefined; + + const data = await signedRequest({ + region: options.region, + method: 'POST', + path: `/configuration-bundles/${options.bundleId}/versions${query ? `?${query}` : ''}`, + body, + }); + + const result = data as ListConfigurationBundleVersionsResult; + return { + versions: result.versions ?? [], + nextToken: result.nextToken, + }; +} diff --git a/src/cli/aws/agentcore-control.ts b/src/cli/aws/agentcore-control.ts index 162c2b3a6..d2770bacf 100644 --- a/src/cli/aws/agentcore-control.ts +++ b/src/cli/aws/agentcore-control.ts @@ -1,4 +1,5 @@ import type { EvaluationLevel } from '../../schema/schemas/primitives/evaluator'; +import { PACKAGE_VERSION } from '../constants'; import { getCredentialProvider } from './account'; import { BedrockAgentCoreControlClient, @@ -27,6 +28,7 @@ export function createControlClient(region: string): BedrockAgentCoreControlClie return new BedrockAgentCoreControlClient({ region, credentials: getCredentialProvider(), + customUserAgent: [['agentcore-cli', PACKAGE_VERSION]], }); } diff --git a/src/cli/aws/agentcore-harness.ts b/src/cli/aws/agentcore-harness.ts new file mode 100644 index 000000000..cca371fbc --- /dev/null +++ b/src/cli/aws/agentcore-harness.ts @@ -0,0 +1,625 @@ +/** + * Typed client wrappers for Harness control plane and data plane operations. + * + * Control plane: CreateHarness, GetHarness, UpdateHarness, DeleteHarness, ListHarnesses + * Data plane: InvokeHarness (streaming) + * TODO InvokeAgentRuntimeCommand + * + * Built on AgentCoreApiClient (shared SigV4 HTTP client). + * Migrate to @aws-sdk/client-bedrock-agentcore-control when Harness commands land in the SDK. + */ +import { AgentCoreApiClient, AgentCoreApiError, resolveEndpoint } from './api-client'; +import { randomUUID } from 'node:crypto'; + +// ============================================================================ +// Shared Types (from Smithy service model) +// ============================================================================ + +export type HarnessStatus = 'CREATING' | 'READY' | 'UPDATING' | 'DELETING' | 'DELETED' | 'FAILED'; + +export interface HarnessModelConfiguration { + bedrockModelConfig?: { modelId: string }; + openAiModelConfig?: { modelId: string; apiKeyArn?: string }; + geminiModelConfig?: { modelId: string; apiKeyArn?: string }; +} + +export type HarnessSystemPrompt = { text: string }[]; + +export interface HarnessTool { + type: string; + name: string; + browserArn?: string; + codeInterpreterArn?: string; + config?: Record; +} + +export interface HarnessSkill { + path: string; +} + +export interface HarnessMemoryConfiguration { + memoryArn?: string; +} + +export interface HarnessTruncationConfiguration { + strategy: string; + config: { slidingWindow?: { messagesCount: number } }; +} + +export interface HarnessEnvironmentArtifact { + containerConfiguration?: { containerUri: string }; +} + +export interface HarnessAgentCoreRuntimeEnvironment { + agentRuntimeArn?: string; + agentRuntimeId?: string; + agentRuntimeName?: string; + lifecycleConfiguration?: Record; + networkConfiguration?: Record; + filesystemConfigurations?: Record[]; +} + +export interface HarnessEnvironmentProvider { + agentCoreRuntimeEnvironment?: HarnessAgentCoreRuntimeEnvironment; +} + +export interface Harness { + harnessId: string; + harnessName: string; + arn: string; + status: HarnessStatus; + executionRoleArn: string; + model?: HarnessModelConfiguration; + systemPrompt?: HarnessSystemPrompt; + tools?: HarnessTool[]; + skills?: HarnessSkill[]; + allowedTools?: string[]; + memory?: HarnessMemoryConfiguration; + truncation?: HarnessTruncationConfiguration; + maxIterations?: number; + maxTokens?: number; + timeoutSeconds?: number; + environment?: HarnessEnvironmentProvider; + environmentArtifact?: HarnessEnvironmentArtifact; + environmentVariables?: Record; + authorizerConfiguration?: Record; + tags?: Record; + createdAt: string; + updatedAt: string; +} + +export interface HarnessSummary { + harnessId: string; + harnessName: string; + arn: string; + status: HarnessStatus; + createdAt: string; + updatedAt: string; +} + +// ============================================================================ +// CreateHarness +// ============================================================================ + +export interface CreateHarnessOptions { + region: string; + harnessName: string; + executionRoleArn: string; + environment?: HarnessEnvironmentProvider; + environmentArtifact?: HarnessEnvironmentArtifact; + environmentVariables?: Record; + authorizerConfiguration?: Record; + model?: HarnessModelConfiguration; + systemPrompt?: HarnessSystemPrompt; + tools?: HarnessTool[]; + skills?: HarnessSkill[]; + allowedTools?: string[]; + memory?: HarnessMemoryConfiguration; + truncation?: HarnessTruncationConfiguration; + maxIterations?: number; + maxTokens?: number; + timeoutSeconds?: number; + tags?: Record; +} + +export interface CreateHarnessResult { + harness: Harness; +} + +export async function createHarness(options: CreateHarnessOptions): Promise { + const { region, ...rest } = options; + const client = new AgentCoreApiClient({ region, plane: 'control' }); + + const body: Record = { + harnessName: rest.harnessName, + clientToken: randomUUID(), + executionRoleArn: rest.executionRoleArn, + }; + + if (rest.environment) body.environment = rest.environment; + if (rest.environmentArtifact) body.environmentArtifact = rest.environmentArtifact; + if (rest.environmentVariables) body.environmentVariables = rest.environmentVariables; + if (rest.authorizerConfiguration) body.authorizerConfiguration = rest.authorizerConfiguration; + if (rest.model) body.model = rest.model; + if (rest.systemPrompt) body.systemPrompt = rest.systemPrompt; + if (rest.tools) body.tools = rest.tools; + if (rest.skills) body.skills = rest.skills; + if (rest.allowedTools) body.allowedTools = rest.allowedTools; + if (rest.memory) body.memory = rest.memory; + if (rest.truncation) body.truncation = rest.truncation; + if (rest.maxIterations != null) body.maxIterations = rest.maxIterations; + if (rest.maxTokens != null) body.maxTokens = rest.maxTokens; + if (rest.timeoutSeconds != null) body.timeoutSeconds = rest.timeoutSeconds; + if (rest.tags) body.tags = rest.tags; + + const result = await client.request({ method: 'POST', path: '/harnesses', body }); + return result as CreateHarnessResult; +} + +// ============================================================================ +// GetHarness +// ============================================================================ + +export interface GetHarnessOptions { + region: string; + harnessId: string; +} + +export interface GetHarnessResult { + harness: Harness; +} + +export async function getHarness(options: GetHarnessOptions): Promise { + const client = new AgentCoreApiClient({ region: options.region, plane: 'control' }); + const result = await client.request({ method: 'GET', path: `/harnesses/${options.harnessId}` }); + return result as GetHarnessResult; +} + +// ============================================================================ +// UpdateHarness +// ============================================================================ + +export interface UpdateHarnessOptions { + region: string; + harnessId: string; + executionRoleArn?: string; + environment?: HarnessEnvironmentProvider; + environmentArtifact?: { optionalValue: HarnessEnvironmentArtifact | null }; + environmentVariables?: Record; + authorizerConfiguration?: { optionalValue: Record | null }; + model?: HarnessModelConfiguration; + systemPrompt?: HarnessSystemPrompt; + tools?: HarnessTool[]; + skills?: HarnessSkill[]; + allowedTools?: string[]; + memory?: { optionalValue: HarnessMemoryConfiguration | null }; + truncation?: HarnessTruncationConfiguration; + maxIterations?: number; + maxTokens?: number; + timeoutSeconds?: number; + tags?: Record; +} + +export interface UpdateHarnessResult { + harness: Harness; +} + +export async function updateHarness(options: UpdateHarnessOptions): Promise { + const { region, harnessId, ...rest } = options; + const client = new AgentCoreApiClient({ region, plane: 'control' }); + + const body: Record = { + clientToken: randomUUID(), + }; + + if (rest.executionRoleArn) body.executionRoleArn = rest.executionRoleArn; + if (rest.environment) body.environment = rest.environment; + if (rest.environmentArtifact !== undefined) body.environmentArtifact = rest.environmentArtifact; + if (rest.environmentVariables) body.environmentVariables = rest.environmentVariables; + if (rest.authorizerConfiguration !== undefined) body.authorizerConfiguration = rest.authorizerConfiguration; + if (rest.model) body.model = rest.model; + if (rest.systemPrompt) body.systemPrompt = rest.systemPrompt; + if (rest.tools) body.tools = rest.tools; + if (rest.skills) body.skills = rest.skills; + if (rest.allowedTools) body.allowedTools = rest.allowedTools; + if (rest.memory !== undefined) body.memory = rest.memory; + if (rest.truncation) body.truncation = rest.truncation; + if (rest.maxIterations != null) body.maxIterations = rest.maxIterations; + if (rest.maxTokens != null) body.maxTokens = rest.maxTokens; + if (rest.timeoutSeconds != null) body.timeoutSeconds = rest.timeoutSeconds; + if (rest.tags) body.tags = rest.tags; + + const result = await client.request({ method: 'PATCH', path: `/harnesses/${harnessId}`, body }); + return result as UpdateHarnessResult; +} + +// ============================================================================ +// DeleteHarness +// ============================================================================ + +export interface DeleteHarnessOptions { + region: string; + harnessId: string; +} + +export interface DeleteHarnessResult { + harness: Harness; +} + +export async function deleteHarness(options: DeleteHarnessOptions): Promise { + const client = new AgentCoreApiClient({ region: options.region, plane: 'control' }); + const result = await client.request({ + method: 'DELETE', + path: `/harnesses/${options.harnessId}`, + query: { clientToken: randomUUID() }, + }); + return result as DeleteHarnessResult; +} + +// ============================================================================ +// ListHarnesses +// ============================================================================ + +export interface ListHarnessesOptions { + region: string; + maxResults?: number; + nextToken?: string; +} + +export interface ListHarnessesResult { + harnesses: HarnessSummary[]; + nextToken?: string; +} + +export async function listHarnesses(options: ListHarnessesOptions): Promise { + const client = new AgentCoreApiClient({ region: options.region, plane: 'control' }); + const query: Record = {}; + if (options.maxResults != null) query.maxResults = String(options.maxResults); + if (options.nextToken) query.nextToken = options.nextToken; + + const result = await client.request({ method: 'GET', path: '/harnesses', query }); + return result as ListHarnessesResult; +} + +export async function listAllHarnesses(region: string): Promise { + const all: HarnessSummary[] = []; + let nextToken: string | undefined; + + do { + const result = await listHarnesses({ region, maxResults: 100, nextToken }); + all.push(...result.harnesses); + nextToken = result.nextToken; + } while (nextToken); + + return all; +} + +// ============================================================================ +// InvokeHarness (streaming, data plane) +// ============================================================================ + +export interface InvokeHarnessOptions { + region: string; + harnessArn: string; + runtimeSessionId: string; + messages: { role: string; content: Record[] }[]; + model?: HarnessModelConfiguration; + systemPrompt?: HarnessSystemPrompt; + tools?: HarnessTool[]; + skills?: HarnessSkill[]; + allowedTools?: string[]; + maxIterations?: number; + maxTokens?: number; + timeoutSeconds?: number; + actorId?: string; + /** Bearer token for CUSTOM_JWT auth (bypasses SigV4) */ + bearerToken?: string; +} + +// ── Stream event types ────────────────────────────────────────────────────── + +export type HarnessStopReason = + | 'end_turn' + | 'tool_use' + | 'tool_result' + | 'max_tokens' + | 'stop_sequence' + | 'content_filtered' + | 'malformed_model_output' + | 'malformed_tool_use' + | 'interrupted' + | 'partial_turn' + | 'model_context_window_exceeded' + | 'max_iterations_exceeded' + | 'max_output_tokens_exceeded' + | 'timeout_exceeded'; + +export interface ToolUseBlockStart { + toolUseId: string; + name: string; + type?: string; + serverName?: string; +} + +export interface ToolResultBlockStart { + toolUseId: string; + status?: string; +} + +export type ContentBlockStart = + | { type: 'toolUse'; toolUse: ToolUseBlockStart } + | { type: 'toolResult'; toolResult: ToolResultBlockStart }; + +export type ContentBlockDelta = + | { type: 'text'; text: string } + | { type: 'toolUse'; input: string } + | { type: 'toolResult'; results: Record[] } + | { type: 'reasoningContent'; text?: string; signature?: string }; + +export interface TokenUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cacheReadInputTokens?: number; + cacheWriteInputTokens?: number; +} + +export interface StreamMetrics { + latencyMs: number; +} + +export type HarnessStreamEvent = + | { type: 'messageStart'; role: string } + | { type: 'contentBlockStart'; contentBlockIndex: number; start: ContentBlockStart } + | { type: 'contentBlockDelta'; contentBlockIndex: number; delta: ContentBlockDelta } + | { type: 'contentBlockStop'; contentBlockIndex: number } + | { type: 'messageStop'; stopReason: HarnessStopReason } + | { type: 'metadata'; usage: TokenUsage; metrics: StreamMetrics } + | { type: 'error'; errorType: string; message: string }; + +export async function* invokeHarness(options: InvokeHarnessOptions): AsyncGenerator { + const { region, harnessArn, runtimeSessionId, messages, bearerToken, ...overrides } = options; + + const body: Record = { messages }; + if (overrides.model) body.model = overrides.model; + if (overrides.systemPrompt) body.systemPrompt = overrides.systemPrompt; + if (overrides.tools) body.tools = overrides.tools; + if (overrides.skills) body.skills = overrides.skills; + if (overrides.allowedTools) body.allowedTools = overrides.allowedTools; + if (overrides.maxIterations != null) body.maxIterations = overrides.maxIterations; + if (overrides.maxTokens != null) body.maxTokens = overrides.maxTokens; + if (overrides.timeoutSeconds != null) body.timeoutSeconds = overrides.timeoutSeconds; + if (overrides.actorId) body.actorId = overrides.actorId; + + let response: Response; + if (bearerToken) { + response = await invokeHarnessWithBearerToken(region, harnessArn, runtimeSessionId, body, bearerToken); + } else { + const client = new AgentCoreApiClient({ region, plane: 'data' }); + response = await client.requestRaw({ + method: 'POST', + path: '/harnesses/invoke', + query: { harnessArn }, + headers: { 'X-Amzn-Bedrock-AgentCore-Runtime-Session-Id': runtimeSessionId }, + body, + }); + } + + if (!response.ok) { + const errorBody = await response.text(); + const requestId = response.headers.get('x-amzn-requestid') ?? undefined; + throw new AgentCoreApiError(response.status, errorBody, requestId); + } + + if (!response.body) return; + + yield* parseEventStream(response.body); +} + +async function invokeHarnessWithBearerToken( + region: string, + harnessArn: string, + runtimeSessionId: string, + body: Record, + bearerToken: string +): Promise { + const endpoint = resolveEndpoint(region, 'data'); + const url = new URL('/harnesses/invoke', endpoint); + url.searchParams.set('harnessArn', harnessArn); + + return fetch(url.toString(), { + method: 'POST', + headers: { + Authorization: `Bearer ${bearerToken}`, + 'Content-Type': 'application/json', + 'X-Amzn-Bedrock-AgentCore-Runtime-Session-Id': runtimeSessionId, + }, + body: JSON.stringify(body), + }); +} + +async function* parseEventStream(body: ReadableStream): AsyncGenerator { + const { EventStreamCodec } = await import('@smithy/eventstream-codec'); + const codec = new EventStreamCodec(toUtf8, fromUtf8); + const reader = body.getReader(); + let buffer: Uint8Array = new Uint8Array(0); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer = concatBuffers(buffer, new Uint8Array(value)); + + while (buffer.length >= 4) { + const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength); + const totalLength = view.getUint32(0); + if (buffer.length < totalLength) break; + + const frame = buffer.slice(0, totalLength); + buffer = buffer.slice(totalLength); + + try { + const message = codec.decode(frame); + const headers: Record = {}; + for (const [key, val] of Object.entries(message.headers)) { + headers[key] = String(val.value); + } + + if (headers[':message-type'] === 'error') { + yield { + type: 'error', + errorType: headers[':error-code'] ?? 'unknown', + message: headers[':error-message'] ?? 'Unknown error', + }; + continue; + } + + if (headers[':message-type'] === 'exception') { + const exBody = new TextDecoder().decode(message.body); + let msg = exBody; + try { + const parsed = JSON.parse(exBody) as { message?: string }; + msg = parsed.message ?? exBody; + } catch { + // use raw body + } + yield { + type: 'error', + errorType: headers[':exception-type'] ?? 'exception', + message: msg, + }; + continue; + } + + const eventType = headers[':event-type']; + if (!eventType) continue; + + const bodyText = new TextDecoder().decode(message.body); + if (!bodyText) continue; + + const event = parseEventPayload(eventType, bodyText); + if (event) yield event; + } catch { + // skip malformed frames + } + } + } + } finally { + reader.releaseLock(); + } +} + +function toUtf8(input: Uint8Array): string { + return new TextDecoder().decode(input); +} + +function fromUtf8(input: string): Uint8Array { + return new TextEncoder().encode(input); +} + +function concatBuffers(a: Uint8Array, b: Uint8Array): Uint8Array { + const result = new Uint8Array(a.length + b.length); + result.set(a, 0); + result.set(b, a.length); + return result; +} + +function parseEventPayload(eventType: string, bodyText: string): HarnessStreamEvent | null { + let payload: Record; + try { + payload = JSON.parse(bodyText) as Record; + } catch { + return null; + } + + switch (eventType) { + case 'messageStart': + return { type: 'messageStart', role: (payload.role as string) ?? 'assistant' }; + + case 'contentBlockStart': { + const start = (payload.start as Record) ?? payload; + return { + type: 'contentBlockStart', + contentBlockIndex: (payload.contentBlockIndex as number) ?? 0, + start: parseContentBlockStart(start), + }; + } + + case 'contentBlockDelta': { + const delta = (payload.delta as Record) ?? payload; + return { + type: 'contentBlockDelta', + contentBlockIndex: (payload.contentBlockIndex as number) ?? 0, + delta: parseContentBlockDelta(delta), + }; + } + + case 'contentBlockStop': + return { type: 'contentBlockStop', contentBlockIndex: (payload.contentBlockIndex as number) ?? 0 }; + + case 'messageStop': + return { type: 'messageStop', stopReason: (payload.stopReason as HarnessStopReason) ?? 'end_turn' }; + + case 'metadata': + return { + type: 'metadata', + usage: (payload.usage as TokenUsage) ?? { inputTokens: 0, outputTokens: 0, totalTokens: 0 }, + metrics: (payload.metrics as StreamMetrics) ?? { latencyMs: 0 }, + }; + + case 'internalServerException': + return { + type: 'error', + errorType: 'internalServerException', + message: (payload.message as string) ?? 'Internal server error', + }; + + case 'validationException': + return { + type: 'error', + errorType: 'validationException', + message: (payload.message as string) ?? 'Validation error', + }; + + case 'runtimeClientError': + return { + type: 'error', + errorType: 'runtimeClientError', + message: (payload.message as string) ?? 'Runtime client error', + }; + + default: + return null; + } +} + +function parseContentBlockStart(start: Record): ContentBlockStart { + if ('toolUse' in start) { + const tu = start.toolUse as ToolUseBlockStart; + return { type: 'toolUse', toolUse: tu }; + } + if ('toolResult' in start) { + const tr = start.toolResult as ToolResultBlockStart; + return { type: 'toolResult', toolResult: tr }; + } + return { type: 'toolUse', toolUse: { toolUseId: '', name: 'unknown' } }; +} + +function parseContentBlockDelta(delta: Record): ContentBlockDelta { + if ('text' in delta) { + return { type: 'text', text: delta.text as string }; + } + if ('toolUse' in delta) { + const tu = delta.toolUse as { input: string }; + return { type: 'toolUse', input: tu.input }; + } + if ('toolResult' in delta) { + return { type: 'toolResult', results: delta.toolResult as Record[] }; + } + if ('reasoningContent' in delta) { + const rc = delta.reasoningContent as { text?: string; signature?: string }; + return { type: 'reasoningContent', text: rc.text, signature: rc.signature }; + } + return { type: 'text', text: '' }; +} diff --git a/src/cli/aws/agentcore-http-gateways.ts b/src/cli/aws/agentcore-http-gateways.ts new file mode 100644 index 000000000..674f090a0 --- /dev/null +++ b/src/cli/aws/agentcore-http-gateways.ts @@ -0,0 +1,519 @@ +/** + * AWS client wrappers for HTTP Gateway control plane operations. + * + * HTTP gateways are required for A/B testing because MCP gateways + * don't emit spans for treatment propagation. These wrappers use + * direct HTTP requests with SigV4 signing against the control plane. + */ +import { getCredentialProvider } from './account'; +import { dnsSuffix } from './partition'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { HttpRequest } from '@smithy/protocol-http'; +import { SignatureV4 } from '@smithy/signature-v4'; +import { randomUUID } from 'node:crypto'; + +// ============================================================================ +// Types +// ============================================================================ + +// ── Create Gateway ───────────────────────────────────────────────────────── + +export interface CreateHttpGatewayOptions { + region: string; + name: string; + roleArn: string; +} + +export interface CreateHttpGatewayResult { + gatewayId: string; + gatewayArn: string; + name: string; + status: string; +} + +// ── Create Gateway Target ────────────────────────────────────────────────── + +export interface CreateHttpGatewayTargetOptions { + region: string; + gatewayId: string; + targetName: string; + runtimeArn: string; + qualifier?: string; +} + +export interface CreateHttpGatewayTargetResult { + targetId: string; + name: string; + status: string; +} + +// ── Get Gateway ──────────────────────────────────────────────────────────── + +export interface GetHttpGatewayOptions { + region: string; + gatewayId: string; +} + +export interface GetHttpGatewayResult { + gatewayId: string; + gatewayArn: string; + gatewayUrl?: string; + name: string; + status: string; + authorizerType?: string; + roleArn?: string; + createdAt?: string; + updatedAt?: string; +} + +// ── Get Gateway Target ───────────────────────────────────────────────────── + +export interface GetHttpGatewayTargetOptions { + region: string; + gatewayId: string; + targetId: string; +} + +export interface GetHttpGatewayTargetResult { + targetId: string; + name: string; + status: string; + targetConfiguration?: unknown; + createdAt?: string; + updatedAt?: string; +} + +// ── List Gateways ────────────────────────────────────────────────────────── + +export interface ListHttpGatewaysOptions { + region: string; + maxResults?: number; + nextToken?: string; +} + +export interface HttpGatewaySummary { + gatewayId: string; + gatewayArn: string; + name: string; + status: string; +} + +export interface ListHttpGatewaysResult { + gateways: HttpGatewaySummary[]; + nextToken?: string; +} + +// ── List Gateway Targets ────────────────────────────────────────────────── + +export interface ListHttpGatewayTargetsOptions { + region: string; + gatewayId: string; + maxResults?: number; +} + +export interface HttpGatewayTargetSummary { + targetId: string; + name: string; + status: string; +} + +export interface ListHttpGatewayTargetsResult { + targets: HttpGatewayTargetSummary[]; +} + +// ── Delete Gateway Target ────────────────────────────────────────────────── + +export interface DeleteHttpGatewayTargetOptions { + region: string; + gatewayId: string; + targetId: string; +} + +// ── Delete Gateway ───────────────────────────────────────────────────────── + +export interface DeleteHttpGatewayOptions { + region: string; + gatewayId: string; +} + +// ── Wait for Target Ready ────────────────────────────────────────────────── + +export interface WaitForTargetReadyOptions { + region: string; + gatewayId: string; + targetId: string; + /** Maximum time to wait in milliseconds. Defaults to 120000 (120s). */ + timeoutMs?: number; +} + +// ============================================================================ +// HTTP signing helper +// ============================================================================ + +function getControlPlaneEndpoint(region: string): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapcp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapcp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore-control.${region}.${dnsSuffix(region)}`; +} + +async function signedRequest(options: { + region: string; + method: string; + path: string; + body?: string; +}): Promise { + const { region, method, path, body } = options; + const endpoint = getControlPlaneEndpoint(region); + const url = new URL(path, endpoint); + + const query: Record = {}; + url.searchParams.forEach((value, key) => { + query[key] = value; + }); + + const request = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname, + ...(Object.keys(query).length > 0 && { query }), + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + }, + ...(body && { body }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const service = 'bedrock-agentcore'; + const signer = new SignatureV4({ + service, + region, + credentials, + sha256: Sha256, + }); + + const signedReq = await signer.sign(request); + + const response = await fetch(`${endpoint}${path}`, { + method, + headers: signedReq.headers as Record, + ...(body && { body }), + }); + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error(`HttpGateway API error (${response.status}): ${errorBody}`); + } + + if (response.status === 204) return {}; + return response.json(); +} + +// ============================================================================ +// Control Plane Operations +// ============================================================================ + +export async function createHttpGateway(options: CreateHttpGatewayOptions): Promise { + const body = JSON.stringify({ + name: options.name, + authorizerType: 'AWS_IAM', + roleArn: options.roleArn, + clientToken: randomUUID(), + }); + + try { + return (await signedRequest({ + region: options.region, + method: 'POST', + path: '/gateways', + body, + })) as CreateHttpGatewayResult; + } catch (err) { + throw new Error( + `Failed to create HTTP gateway "${options.name}": ${err instanceof Error ? err.message : String(err)}` + ); + } +} + +export async function createHttpGatewayTarget( + options: CreateHttpGatewayTargetOptions +): Promise { + const body = JSON.stringify({ + name: options.targetName, + clientToken: randomUUID(), + targetConfiguration: { + http: { + agentcoreRuntime: { + arn: options.runtimeArn, + qualifier: options.qualifier ?? 'DEFAULT', + }, + }, + }, + credentialProviderConfigurations: [{ credentialProviderType: 'GATEWAY_IAM_ROLE' }], + }); + + try { + return (await signedRequest({ + region: options.region, + method: 'POST', + path: `/gateways/${options.gatewayId}/targets`, + body, + })) as CreateHttpGatewayTargetResult; + } catch (err) { + // Fallback: retry with legacy field name if the new name is not yet supported + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes('ValidationException') || msg.includes('400')) { + const legacyBody = JSON.stringify({ + name: options.targetName, + clientToken: randomUUID(), + targetConfiguration: { + http: { + runtimeTargetConfiguration: { + arn: options.runtimeArn, + qualifier: options.qualifier ?? 'DEFAULT', + }, + }, + }, + credentialProviderConfigurations: [{ credentialProviderType: 'GATEWAY_IAM_ROLE' }], + }); + try { + return (await signedRequest({ + region: options.region, + method: 'POST', + path: `/gateways/${options.gatewayId}/targets`, + body: legacyBody, + })) as CreateHttpGatewayTargetResult; + } catch { + // Fall through to original error + } + } + throw new Error(`Failed to create target "${options.targetName}" in gateway ${options.gatewayId}: ${msg}`); + } +} + +export async function getHttpGateway(options: GetHttpGatewayOptions): Promise { + const data = await signedRequest({ + region: options.region, + method: 'GET', + path: `/gateways/${options.gatewayId}`, + }); + + return data as GetHttpGatewayResult; +} + +export async function getHttpGatewayTarget(options: GetHttpGatewayTargetOptions): Promise { + const data = await signedRequest({ + region: options.region, + method: 'GET', + path: `/gateways/${options.gatewayId}/targets/${options.targetId}`, + }); + + return data as GetHttpGatewayTargetResult; +} + +export async function listHttpGateways(options: ListHttpGatewaysOptions): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + const query = params.toString(); + + const data = await signedRequest({ + region: options.region, + method: 'GET', + path: `/gateways${query ? `?${query}` : ''}`, + }); + + const result = data as ListHttpGatewaysResult; + return { + gateways: result.gateways ?? [], + nextToken: result.nextToken, + }; +} + +/** + * List all HTTP gateways, paginating through all results. + */ +export async function listAllHttpGateways(options: { region: string }): Promise { + const all: HttpGatewaySummary[] = []; + let nextToken: string | undefined; + + do { + const result = await listHttpGateways({ region: options.region, maxResults: 100, nextToken }); + all.push(...result.gateways); + nextToken = result.nextToken; + } while (nextToken); + + return all; +} + +export async function listHttpGatewayTargets( + options: ListHttpGatewayTargetsOptions +): Promise { + const params = new URLSearchParams(); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + const query = params.toString(); + + const data = await signedRequest({ + region: options.region, + method: 'GET', + path: `/gateways/${options.gatewayId}/targets${query ? `?${query}` : ''}`, + }); + + const result = data as Record; + return { + targets: (result.items ?? result.targets ?? []) as HttpGatewayTargetSummary[], + }; +} + +export async function deleteHttpGatewayTarget( + options: DeleteHttpGatewayTargetOptions +): Promise<{ success: boolean; error?: string }> { + try { + await signedRequest({ + region: options.region, + method: 'DELETE', + path: `/gateways/${options.gatewayId}/targets/${options.targetId}`, + }); + + // Wait for target to be fully deleted before returning. + // Gateway deletion fails if targets still exist in DELETING state. + const timeoutMs = 60_000; + const startTime = Date.now(); + let delayMs = 2_000; + + while (Date.now() - startTime < timeoutMs) { + try { + await getHttpGatewayTarget({ + region: options.region, + gatewayId: options.gatewayId, + targetId: options.targetId, + }); + // Target still exists — keep waiting + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes('(404)') || msg.includes('not found')) { + return { success: true }; // Target confirmed deleted + } + // Transient error — keep polling + } + + const remaining = timeoutMs - (Date.now() - startTime); + if (remaining <= 0) break; + await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remaining))); + delayMs = Math.min(delayMs * 2, 8_000); + } + + // Polling timed out — target may still be deleting + return { success: false, error: `Timed out waiting for target ${options.targetId} to be fully deleted` }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +export async function deleteHttpGateway( + options: DeleteHttpGatewayOptions +): Promise<{ success: boolean; error?: string }> { + try { + await signedRequest({ + region: options.region, + method: 'DELETE', + path: `/gateways/${options.gatewayId}`, + }); + return { success: true }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } +} + +/** Terminal states that indicate a resource will never become READY. */ +const TERMINAL_FAILURE_STATES = ['FAILED', 'CREATE_FAILED', 'UPDATE_FAILED', 'DELETING', 'DELETED'] as const; + +export async function waitForGatewayReady(options: { + region: string; + gatewayId: string; + timeoutMs?: number; +}): Promise { + const timeoutMs = options.timeoutMs ?? 120_000; + const startTime = Date.now(); + let delayMs = 2_000; + + while (Date.now() - startTime < timeoutMs) { + const gateway = await getHttpGateway({ + region: options.region, + gatewayId: options.gatewayId, + }); + + if (gateway.status === 'READY') return gateway; + + if ((TERMINAL_FAILURE_STATES as readonly string[]).includes(gateway.status)) { + throw new Error( + `Gateway ${options.gatewayId} reached terminal state '${gateway.status}' and will not become READY` + ); + } + + const remaining = timeoutMs - (Date.now() - startTime); + if (remaining <= 0) break; + + await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remaining))); + delayMs = Math.min(delayMs * 2, 16_000); + } + + throw new Error( + `Timed out waiting for gateway ${options.gatewayId} to become READY after ${Math.round(timeoutMs / 1000)}s` + ); +} + +export async function waitForTargetReady(options: WaitForTargetReadyOptions): Promise { + const timeoutMs = options.timeoutMs ?? 120_000; + const startTime = Date.now(); + let delayMs = 2_000; + + while (Date.now() - startTime < timeoutMs) { + let target; + try { + target = await getHttpGatewayTarget({ + region: options.region, + gatewayId: options.gatewayId, + targetId: options.targetId, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (msg.includes('(404)')) { + throw new Error( + `Target ${options.targetId} not found during readiness poll — it may have been deleted externally` + ); + } + // Retry on transient server errors + if (/\(5\d\d\)/.test(msg)) { + // Continue polling — transient error + const remaining = timeoutMs - (Date.now() - startTime); + if (remaining <= 0) break; + await new Promise(resolve => setTimeout(resolve, delayMs)); + delayMs = Math.min(delayMs * 2, 16_000); + continue; + } + throw err; + } + + if (target.status === 'READY') return target; + + if ((TERMINAL_FAILURE_STATES as readonly string[]).includes(target.status)) { + throw new Error( + `Target ${options.targetId} in gateway ${options.gatewayId} reached terminal state '${target.status}' and will not become READY` + ); + } + + const remaining = timeoutMs - (Date.now() - startTime); + if (remaining <= 0) break; + + await new Promise(resolve => setTimeout(resolve, Math.min(delayMs, remaining))); + delayMs = Math.min(delayMs * 2, 16_000); + } + + throw new Error( + `Timed out waiting for target ${options.targetId} to become READY after ${Math.round(timeoutMs / 1000)}s` + ); +} diff --git a/src/cli/aws/agentcore-recommendation.ts b/src/cli/aws/agentcore-recommendation.ts new file mode 100644 index 000000000..55242fdcd --- /dev/null +++ b/src/cli/aws/agentcore-recommendation.ts @@ -0,0 +1,371 @@ +/** + * AWS client wrappers for Recommendation API operations. + * + * NOTE: The Recommendation API is not yet available in the AWS SDK. + * These wrappers use direct HTTP requests with SigV4 signing as an + * interim solution. When the SDK adds Recommendation commands, migrate + * to the SDK client. + * + * TEMPORARY: All Recommendation endpoints are on the Data Plane (DP), + * not the Control Plane. This is the current API shape as of 2026-03-30. + * The API may move to CP in the future — update endpoints accordingly. + * + * Recommendations are one-shot, immutable resources. There is no Update + * operation and no runs sub-resource. You start a recommendation with + * StartRecommendation, poll via GetRecommendation, and stop via + * DeleteRecommendation (stop-via-delete pattern). + */ +import { getCredentialProvider } from './account'; +import { dnsSuffix } from './partition'; +import { Sha256 } from '@aws-crypto/sha256-js'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; +import { HttpRequest } from '@smithy/protocol-http'; +import { SignatureV4 } from '@smithy/signature-v4'; + +// ============================================================================ +// Types — Recommendation Type Enum +// ============================================================================ + +export type RecommendationType = 'SYSTEM_PROMPT_RECOMMENDATION' | 'TOOL_DESCRIPTION_RECOMMENDATION'; + +// ============================================================================ +// Types — Input Config (tag-union per type) +// ============================================================================ + +/** System prompt source — either inline text or a ConfigBundle reference. */ +export interface SystemPromptSource { + text?: string; + configurationBundle?: { + bundleArn: string; + versionId?: string; + systemPromptJsonPath?: string; + }; +} + +/** A single OTEL-style span for inline session traces. */ +export interface SessionSpan { + scope?: { name: string }; + body?: { + input?: { messages?: { content: unknown; role: string }[] }; + output?: { messages?: { content: unknown; role: string }[] }; + }; + attributes?: Record; + traceId: string; + spanId: string; +} + +/** Agent trace source — inline spans or CloudWatch Logs. */ +export interface AgentTracesSource { + sessionSpans?: SessionSpan[]; + cloudwatchLogs?: { + logGroupArns: string[]; + serviceNames: string[]; + startTime: string; + endTime: string; + limit?: number; + sessionIds?: string[]; + }; +} + +/** Evaluation config — exactly one evaluator as objective signal (API constraint: min 1, max 1). */ +export interface RecommendationEvaluationConfig { + evaluators: [{ evaluatorArn: string }]; +} + +/** Config for SYSTEM_PROMPT_RECOMMENDATION type. */ +export interface SystemPromptRecommendationConfig { + systemPrompt: SystemPromptSource; + agentTraces: AgentTracesSource; + evaluationConfig: RecommendationEvaluationConfig; +} + +/** Config for TOOL_DESCRIPTION_RECOMMENDATION type. */ +export interface ToolDescriptionRecommendationConfig { + toolDescription: { + toolDescriptionText?: { + tools: { toolName: string; toolDescription: { text: string } }[]; + }; + configurationBundle?: { + bundleArn: string; + versionId?: string; + tools: { toolName: string; toolDescriptionJsonPath: string }[]; + }; + }; + agentTraces: AgentTracesSource; +} + +/** Tag-union recommendation config — only populate the member matching the type. */ +export interface RecommendationConfig { + systemPromptRecommendationConfig?: SystemPromptRecommendationConfig; + toolDescriptionRecommendationConfig?: ToolDescriptionRecommendationConfig; +} + +// ============================================================================ +// Types — Result (tag-union per type) +// ============================================================================ + +export interface RecommendationResultConfigurationBundle { + bundleArn: string; + versionId: string; +} + +export interface SystemPromptRecommendationResult { + recommendedSystemPrompt?: string; + configurationBundle?: RecommendationResultConfigurationBundle; + errorCode?: string; + errorMessage?: string; +} + +export interface ToolDescriptionRecommendationToolResult { + toolName: string; + recommendedToolDescription: string; +} + +export interface ToolDescriptionRecommendationResult { + tools?: ToolDescriptionRecommendationToolResult[]; + configurationBundle?: RecommendationResultConfigurationBundle; + errorCode?: string; + errorMessage?: string; +} + +export interface RecommendationResult { + systemPromptRecommendationResult?: SystemPromptRecommendationResult; + toolDescriptionRecommendationResult?: ToolDescriptionRecommendationResult; +} + +// ============================================================================ +// Types — API Options & Results +// ============================================================================ + +export interface StartRecommendationOptions { + region: string; + name: string; + description?: string; + type: RecommendationType; + recommendationConfig: RecommendationConfig; + kmsKeyArn?: string; + clientToken?: string; +} + +export interface StartRecommendationResult { + recommendationId: string; + recommendationArn: string; + name: string; + type: string; + status: string; + createdAt?: string; + updatedAt?: string; + requestId?: string; +} + +export interface GetRecommendationOptions { + region: string; + recommendationId: string; +} + +export interface GetRecommendationResult { + recommendationId: string; + recommendationArn: string; + name: string; + description?: string; + type: string; + recommendationConfig?: RecommendationConfig; + status: string; + statusReasons?: string[]; + createdAt?: string; + updatedAt?: string; + completedAt?: string; + recommendationResult?: RecommendationResult; + requestId?: string; +} + +export interface ListRecommendationsOptions { + region: string; + status?: string; + maxResults?: number; + nextToken?: string; +} + +export interface RecommendationSummary { + recommendationId: string; + recommendationArn: string; + name: string; + description?: string; + type: string; + status: string; + createdAt?: string; + updatedAt?: string; +} + +export interface ListRecommendationsResult { + recommendationSummaries: RecommendationSummary[]; + nextToken?: string; +} + +export interface DeleteRecommendationOptions { + region: string; + recommendationId: string; +} + +export interface DeleteRecommendationResult { + recommendationId: string; + status: string; +} + +// ============================================================================ +// HTTP signing helper +// ============================================================================ + +/** + * Resolve the DP endpoint for the Recommendation API. + * + * TEMPORARY: All Recommendation endpoints are on the Data Plane. + * Set AGENTCORE_STAGE=beta|gamma to target pre-release environments. + */ +function getDataPlaneEndpoint(region: string): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapdp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapdp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore.${region}.${dnsSuffix(region)}`; +} + +async function signedRequest(options: { + region: string; + method: string; + path: string; + body?: string; +}): Promise<{ data: unknown; status: number; requestId?: string }> { + const { region, method, path, body } = options; + const endpoint = getDataPlaneEndpoint(region); + const url = new URL(path, endpoint); + + const query: Record = {}; + url.searchParams.forEach((value, key) => { + query[key] = value; + }); + + const request = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname, + ...(Object.keys(query).length > 0 && { query }), + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + }, + ...(body && { body }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const signer = new SignatureV4({ + service: 'bedrock-agentcore', + region, + credentials, + sha256: Sha256, + }); + + const signedReq = await signer.sign(request); + + const response = await fetch(`${endpoint}${path}`, { + method, + headers: signedReq.headers as Record, + ...(body && { body }), + }); + + const requestId = response.headers.get('x-amzn-requestid') ?? 'unknown'; + + if (!response.ok) { + const errorBody = await response.text(); + throw new Error(`Recommendation API error (${response.status}): ${errorBody} [requestId: ${requestId}]`); + } + + if (response.status === 204) return { data: {}, status: 204, requestId }; + return { data: await response.json(), status: response.status, requestId }; +} + +// ============================================================================ +// API Operations +// ============================================================================ + +/** + * Start a new recommendation (async — returns 202). + * Creates an ARN-able resource that progresses through: + * PENDING → IN_PROGRESS → COMPLETED | FAILED + */ +export async function startRecommendation(options: StartRecommendationOptions): Promise { + const body = JSON.stringify({ + name: options.name, + ...(options.description && { description: options.description }), + type: options.type, + recommendationConfig: options.recommendationConfig, + ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }), + ...(options.clientToken && { clientToken: options.clientToken }), + }); + + const { data, requestId } = await signedRequest({ + region: options.region, + method: 'POST', + path: '/recommendations', + body, + }); + + const result = data as StartRecommendationResult; + if (requestId) result.requestId = requestId; + return result; +} + +/** + * Get recommendation status and results. + * When status is COMPLETED, recommendationResult contains the optimized artifact. + */ +export async function getRecommendation(options: GetRecommendationOptions): Promise { + const { data, requestId } = await signedRequest({ + region: options.region, + method: 'GET', + path: `/recommendations/${options.recommendationId}`, + }); + + const result = data as GetRecommendationResult; + if (requestId) result.requestId = requestId; + return result; +} + +/** + * List recommendations with optional filtering and pagination. + */ +export async function listRecommendations(options: ListRecommendationsOptions): Promise { + const params = new URLSearchParams(); + if (options.status) params.set('status', options.status); + if (options.maxResults) params.set('maxResults', String(options.maxResults)); + if (options.nextToken) params.set('nextToken', options.nextToken); + + const query = params.toString(); + const path = `/recommendations${query ? `?${query}` : ''}`; + + const { data } = await signedRequest({ + region: options.region, + method: 'GET', + path, + }); + + const result = data as ListRecommendationsResult; + return { + recommendationSummaries: result.recommendationSummaries ?? [], + nextToken: result.nextToken, + }; +} + +/** + * Delete a recommendation. Also stops in-progress recommendations + * (stop-via-delete pattern — no separate Stop API). + */ +export async function deleteRecommendation(options: DeleteRecommendationOptions): Promise { + const { data } = await signedRequest({ + region: options.region, + method: 'DELETE', + path: `/recommendations/${options.recommendationId}`, + }); + + return data as DeleteRecommendationResult; +} diff --git a/src/cli/aws/agentcore.ts b/src/cli/aws/agentcore.ts index 55c19d2d0..4c50a0330 100644 --- a/src/cli/aws/agentcore.ts +++ b/src/cli/aws/agentcore.ts @@ -1,11 +1,11 @@ import { parseJsonRpcResponse } from '../../lib/utils/json-rpc'; +import { PACKAGE_VERSION } from '../constants'; import { getCredentialProvider } from './account'; import { parseAguiSSEStream } from './agui-parser'; import { serviceEndpoint } from './partition'; import { BedrockAgentCoreClient, EvaluateCommand, - type EvaluationReferenceInput, InvokeAgentRuntimeCommand, InvokeAgentRuntimeCommandCommand, StopRuntimeSessionCommand, @@ -13,13 +13,31 @@ import { import type { HttpRequest } from '@smithy/protocol-http'; import type { DocumentType } from '@smithy/types'; +/** Local definition — SDK does not yet export this type. */ +export interface EvaluationReferenceInput { + context: { spanContext: { sessionId: string; traceId?: string } }; + expectedTrajectory?: { toolNames: string[] }; + assertions?: { text: string }[]; + expectedResponse?: { text: string }; +} + /** * Create a BedrockAgentCoreClient with optional custom header injection middleware. */ -function createAgentCoreClient(region: string, headers?: Record): BedrockAgentCoreClient { +function resolveDataPlaneEndpoint(region: string): string | undefined { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + if (stage === 'beta') return `https://beta.${region}.elcapdp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapdp.genesis-primitives.aws.dev`; + return undefined; +} + +export function createAgentCoreClient(region: string, headers?: Record): BedrockAgentCoreClient { + const endpoint = resolveDataPlaneEndpoint(region); const client = new BedrockAgentCoreClient({ region, credentials: getCredentialProvider(), + customUserAgent: [['agentcore-cli', PACKAGE_VERSION]], + ...(endpoint && { endpoint }), }); if (headers && Object.keys(headers).length > 0) { @@ -59,6 +77,8 @@ export interface InvokeAgentRuntimeOptions { headers?: Record; /** Bearer token for CUSTOM_JWT auth. When provided, uses raw HTTP with Authorization header instead of SigV4. */ bearerToken?: string; + /** W3C baggage header value (e.g. config bundle ref for runtime) */ + baggage?: string; } export interface InvokeAgentRuntimeResult { @@ -151,20 +171,40 @@ function buildInvokeUrl(region: string, runtimeArn: string): string { } /** - * Invoke an AgentCore Runtime using bearer token auth (raw HTTP, no SigV4). - * Used when the runtime has CUSTOM_JWT authorizer configured. + * Build headers for bearer-token invoke requests. + * Shared by both streaming and non-streaming invoke paths. */ -async function invokeWithBearerTokenStreaming(options: InvokeAgentRuntimeOptions): Promise { - const url = buildInvokeUrl(options.region, options.runtimeArn); +export function buildBearerInvokeHeaders( + options: Pick, + accept: string +): Record { const headers: Record = { Authorization: `Bearer ${options.bearerToken}`, 'Content-Type': 'application/json', - Accept: 'application/json, text/event-stream', + Accept: accept, }; if (options.sessionId) { headers['X-Amzn-Bedrock-AgentCore-Runtime-Session-Id'] = options.sessionId; } headers['X-Amzn-Bedrock-AgentCore-Runtime-User-Id'] = options.userId ?? DEFAULT_RUNTIME_USER_ID; + if (options.baggage) { + headers.baggage = options.baggage; + } + if (options.headers) { + for (const [name, value] of Object.entries(options.headers)) { + headers[name] = value; + } + } + return headers; +} + +/** + * Invoke an AgentCore Runtime using bearer token auth (raw HTTP, no SigV4). + * Used when the runtime has CUSTOM_JWT authorizer configured. + */ +async function invokeWithBearerTokenStreaming(options: InvokeAgentRuntimeOptions): Promise { + const url = buildInvokeUrl(options.region, options.runtimeArn); + const headers = buildBearerInvokeHeaders(options, 'application/json, text/event-stream'); const res = await fetch(url, { method: 'POST', @@ -250,15 +290,7 @@ async function invokeWithBearerTokenStreaming(options: InvokeAgentRuntimeOptions */ async function invokeWithBearerToken(options: InvokeAgentRuntimeOptions): Promise { const url = buildInvokeUrl(options.region, options.runtimeArn); - const headers: Record = { - Authorization: `Bearer ${options.bearerToken}`, - 'Content-Type': 'application/json', - Accept: 'application/json', - }; - if (options.sessionId) { - headers['X-Amzn-Bedrock-AgentCore-Runtime-Session-Id'] = options.sessionId; - } - headers['X-Amzn-Bedrock-AgentCore-Runtime-User-Id'] = options.userId ?? DEFAULT_RUNTIME_USER_ID; + const headers = buildBearerInvokeHeaders(options, 'application/json'); const res = await fetch(url, { method: 'POST', @@ -300,6 +332,7 @@ export async function invokeAgentRuntimeStreaming(options: InvokeAgentRuntimeOpt accept: 'application/json', runtimeSessionId: options.sessionId, runtimeUserId: options.userId ?? DEFAULT_RUNTIME_USER_ID, + ...(options.baggage && { baggage: options.baggage }), }); const response = await client.send(command); @@ -395,6 +428,7 @@ export async function invokeAgentRuntime(options: InvokeAgentRuntimeOptions): Pr accept: 'application/json', runtimeSessionId: options.sessionId, runtimeUserId: options.userId ?? DEFAULT_RUNTIME_USER_ID, + ...(options.baggage && { baggage: options.baggage }), }); const response = await client.send(command); @@ -464,6 +498,7 @@ export async function evaluate(options: EvaluateOptions): Promise; + headers?: Record; +} + +export class AgentCoreApiError extends Error { + readonly statusCode: number; + readonly requestId: string | undefined; + readonly errorBody: string; + + constructor(statusCode: number, errorBody: string, requestId?: string) { + const reqIdSuffix = requestId ? ` [requestId: ${requestId}]` : ''; + super(`AgentCore API error (${statusCode}): ${errorBody}${reqIdSuffix}`); + this.name = 'AgentCoreApiError'; + this.statusCode = statusCode; + this.requestId = requestId; + this.errorBody = errorBody; + } +} + +export class AgentCoreApiClient { + private readonly region: string; + private readonly endpoint: string; + + constructor(options: ApiClientOptions) { + this.region = options.region; + this.endpoint = resolveEndpoint(options.region, options.plane); + } + + async request(options: RequestOptions): Promise { + const response = await this.requestRaw(options); + + if (!response.ok) { + const errorBody = await response.text(); + const requestId = response.headers.get('x-amzn-requestid') ?? undefined; + throw new AgentCoreApiError(response.status, errorBody, requestId); + } + + if (response.status === 204) return {}; + return response.json(); + } + + async requestRaw(options: RequestOptions): Promise { + const { method, path, body, query, headers: extraHeaders } = options; + + const url = new URL(path, this.endpoint); + if (query) { + for (const [key, value] of Object.entries(query)) { + url.searchParams.set(key, value); + } + } + + const queryRecord: Record = {}; + url.searchParams.forEach((value, key) => { + queryRecord[key] = value; + }); + + const serializedBody = body != null ? JSON.stringify(body) : undefined; + + const httpRequest = new HttpRequest({ + method, + protocol: 'https:', + hostname: url.hostname, + path: url.pathname, + ...(Object.keys(queryRecord).length > 0 && { query: queryRecord }), + headers: { + 'Content-Type': 'application/json', + host: url.hostname, + ...extraHeaders, + }, + ...(serializedBody && { body: serializedBody }), + }); + + const credentials = getCredentialProvider() ?? defaultProvider(); + const signer = new SignatureV4({ + service: SERVICE, + region: this.region, + credentials, + sha256: Sha256, + }); + + const signed = await signer.sign(httpRequest); + + const fullUrl = `${this.endpoint}${url.pathname}${url.search}`; + return fetch(fullUrl, { + method, + headers: signed.headers as Record, + ...(serializedBody && { body: serializedBody }), + }); + } +} + +export function resolveEndpoint(region: string, plane: ApiPlane): string { + const stage = process.env.AGENTCORE_STAGE?.toLowerCase(); + + if (plane === 'control') { + if (stage === 'beta') return `https://beta.${region}.elcapcp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapcp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore-control.${region}.${dnsSuffix(region)}`; + } + + if (stage === 'beta') return `https://beta.${region}.elcapdp.genesis-primitives.aws.dev`; + if (stage === 'gamma') return `https://gamma.${region}.elcapdp.genesis-primitives.aws.dev`; + return `https://bedrock-agentcore.${region}.${dnsSuffix(region)}`; +} diff --git a/src/cli/aws/index.ts b/src/cli/aws/index.ts index 5ce91528a..d306a80e1 100644 --- a/src/cli/aws/index.ts +++ b/src/cli/aws/index.ts @@ -26,7 +26,37 @@ export { type GetPolicyGenerationOptions, type GetPolicyGenerationResult, } from './policy-generation'; +export { AgentCoreApiClient, AgentCoreApiError, type ApiClientOptions, type ApiPlane } from './api-client'; +export { pollUntilTerminal, PollTimeoutError, PollFailureError, type PollOptions } from './poll'; export { + createHarness, + getHarness, + updateHarness, + deleteHarness, + listHarnesses, + listAllHarnesses, + invokeHarness, + type Harness, + type HarnessSummary, + type HarnessStatus, + type HarnessStreamEvent, + type HarnessStopReason, + type TokenUsage, + type StreamMetrics, + type CreateHarnessOptions, + type CreateHarnessResult, + type GetHarnessOptions, + type GetHarnessResult, + type UpdateHarnessOptions, + type UpdateHarnessResult, + type DeleteHarnessOptions, + type DeleteHarnessResult, + type ListHarnessesOptions, + type ListHarnessesResult, + type InvokeHarnessOptions, +} from './agentcore-harness'; +export { + createAgentCoreClient, DEFAULT_RUNTIME_USER_ID, executeBashCommand, invokeA2ARuntime, @@ -48,6 +78,24 @@ export { type StopRuntimeSessionOptions, type StopRuntimeSessionResult, } from './agentcore'; +export { + startRecommendation, + getRecommendation, + listRecommendations, + deleteRecommendation, + type StartRecommendationOptions, + type StartRecommendationResult, + type GetRecommendationOptions, + type GetRecommendationResult, + type ListRecommendationsOptions, + type ListRecommendationsResult, + type DeleteRecommendationOptions, + type DeleteRecommendationResult, + type RecommendationSummary, + type RecommendationType, + type RecommendationConfig, + type RecommendationResult, +} from './agentcore-recommendation'; export { AguiEventType, AguiErrorCode, diff --git a/src/cli/aws/policy-generation.ts b/src/cli/aws/policy-generation.ts index da58edf81..86eb76aa5 100644 --- a/src/cli/aws/policy-generation.ts +++ b/src/cli/aws/policy-generation.ts @@ -1,6 +1,5 @@ -import { getCredentialProvider } from './account'; +import { createControlClient } from './agentcore-control'; import { - BedrockAgentCoreControlClient, GetPolicyGenerationCommand, ListPolicyGenerationAssetsCommand, StartPolicyGenerationCommand, @@ -33,10 +32,7 @@ export interface GetPolicyGenerationResult { export async function startPolicyGeneration( options: StartPolicyGenerationOptions ): Promise { - const client = new BedrockAgentCoreControlClient({ - region: options.region, - credentials: getCredentialProvider(), - }); + const client = createControlClient(options.region); const command = new StartPolicyGenerationCommand({ policyEngineId: options.policyEngineId, @@ -57,10 +53,7 @@ export async function startPolicyGeneration( } export async function getPolicyGeneration(options: GetPolicyGenerationOptions): Promise { - const client = new BedrockAgentCoreControlClient({ - region: options.region, - credentials: getCredentialProvider(), - }); + const client = createControlClient(options.region); // Use the SDK waiter to poll until generation completes const waiterResult = await waitUntilPolicyGenerationCompleted( diff --git a/src/cli/aws/poll.ts b/src/cli/aws/poll.ts new file mode 100644 index 000000000..0adfaca23 --- /dev/null +++ b/src/cli/aws/poll.ts @@ -0,0 +1,47 @@ +/** + * Generic polling utility for async AWS resource status transitions. + */ + +export interface PollOptions { + fn: () => Promise; + isTerminal: (result: T) => boolean; + isFailure?: (result: T) => boolean; + getFailureReason?: (result: T) => string; + intervalMs?: number; + maxWaitMs?: number; +} + +export class PollTimeoutError extends Error { + constructor(maxWaitMs: number) { + super(`Polling timed out after ${maxWaitMs}ms`); + this.name = 'PollTimeoutError'; + } +} + +export class PollFailureError extends Error { + constructor(reason: string) { + super(reason); + this.name = 'PollFailureError'; + } +} + +export async function pollUntilTerminal(options: PollOptions): Promise { + const { fn, isTerminal, isFailure, getFailureReason, intervalMs = 3000, maxWaitMs = 120_000 } = options; + const start = Date.now(); + + while (Date.now() - start < maxWaitMs) { + const result = await fn(); + + if (isTerminal(result)) { + if (isFailure?.(result)) { + const reason = getFailureReason?.(result) ?? 'Resource entered a failed state'; + throw new PollFailureError(reason); + } + return result; + } + + await new Promise(resolve => setTimeout(resolve, intervalMs)); + } + + throw new PollTimeoutError(maxWaitMs); +} diff --git a/src/cli/cli.ts b/src/cli/cli.ts index b8100c0d8..98692639c 100644 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -1,4 +1,8 @@ +import { getOrCreateInstallationId } from '../lib/schemas/io/global-config'; +import { registerABTestCommand } from './commands/abtest'; import { registerAdd } from './commands/add'; +import { registerAddTool } from './commands/add/tool-command'; +import { registerConfigBundle } from './commands/config-bundle'; import { registerCreate } from './commands/create'; import { registerDeploy } from './commands/deploy'; import { registerDev } from './commands/dev'; @@ -9,17 +13,19 @@ import { registerImport } from './commands/import'; import { registerInvoke } from './commands/invoke'; import { registerLogs } from './commands/logs'; import { registerPackage } from './commands/package'; -import { registerPause } from './commands/pause'; +import { registerPause, registerPromote } from './commands/pause'; +import { registerRecommendations } from './commands/recommendations'; import { registerRemove } from './commands/remove'; +import { registerRemoveTool } from './commands/remove/tool-command'; import { registerResume } from './commands/resume'; import { registerRun } from './commands/run'; import { registerStatus } from './commands/status'; +import { registerStop } from './commands/stop'; import { registerTelemetry } from './commands/telemetry'; import { registerTraces } from './commands/traces'; import { registerUpdate } from './commands/update'; import { registerValidate } from './commands/validate'; import { PACKAGE_VERSION } from './constants'; -import { getOrCreateInstallationId } from './global-config'; import { ALL_PRIMITIVES } from './primitives'; import { TelemetryClientAccessor } from './telemetry'; import { App } from './tui/App'; @@ -182,19 +188,30 @@ export function registerCommands(program: Command) { registerLogs(program); registerPackage(program); registerPause(program); + registerRecommendations(program); const removeCmd = registerRemove(program); registerResume(program); registerRun(program); registerStatus(program); + registerStop(program); + registerPromote(program); registerTelemetry(program); registerTraces(program); registerUpdate(program); registerValidate(program); + registerConfigBundle(program); // Register primitive subcommands (add agent, remove agent, add memory, etc.) for (const primitive of ALL_PRIMITIVES) { primitive.registerCommands(addCmd, removeCmd); } + + // Register standalone add/remove subcommands + registerAddTool(addCmd); + registerRemoveTool(removeCmd); + + // Register AB test detail command + registerABTestCommand(program); } export const main = async (argv: string[]) => { @@ -208,7 +225,7 @@ export const main = async (argv: string[]) => { const args = argv.slice(2); - // Fire off non-blocking update check (skip for `update` command) + // Fire off non-blocking update check (skip for `update` command itself) const isUpdateCommand = args[0] === 'update'; const updateCheck = isUpdateCommand ? Promise.resolve(null) : checkForUpdate(); diff --git a/src/cli/cloudformation/__tests__/outputs-extended.test.ts b/src/cli/cloudformation/__tests__/outputs-extended.test.ts index cbe82085e..1f48faa96 100644 --- a/src/cli/cloudformation/__tests__/outputs-extended.test.ts +++ b/src/cli/cloudformation/__tests__/outputs-extended.test.ts @@ -364,7 +364,7 @@ describe('parseOnlineEvalOutputs', () => { 'arn:aws:bedrock:us-east-1:123:online-evaluation-config/proj_TestConfig-xyz', }; - const result = parseOnlineEvalOutputs(outputs, ['TestConfig']); + const result = parseOnlineEvalOutputs(outputs, [{ name: 'TestConfig' }]); expect(result.TestConfig).toBeDefined(); expect(result.TestConfig!.onlineEvaluationConfigId).toBe('proj_TestConfig-xyz'); expect(result.TestConfig!.onlineEvaluationConfigArn).toBe( @@ -380,7 +380,7 @@ describe('parseOnlineEvalOutputs', () => { ApplicationOnlineEvalConfigBArnOutputD: 'arn:b', }; - const result = parseOnlineEvalOutputs(outputs, ['ConfigA', 'ConfigB']); + const result = parseOnlineEvalOutputs(outputs, [{ name: 'ConfigA' }, { name: 'ConfigB' }]); expect(Object.keys(result)).toHaveLength(2); expect(result.ConfigA!.onlineEvaluationConfigId).toBe('id-a'); expect(result.ConfigB!.onlineEvaluationConfigId).toBe('id-b'); @@ -391,12 +391,12 @@ describe('parseOnlineEvalOutputs', () => { ApplicationOnlineEvalTestConfigArnOutputDEF: 'arn:config', }; - const result = parseOnlineEvalOutputs(outputs, ['TestConfig']); + const result = parseOnlineEvalOutputs(outputs, [{ name: 'TestConfig' }]); expect(result.TestConfig).toBeUndefined(); }); it('returns empty record for empty outputs', () => { - const result = parseOnlineEvalOutputs({}, ['TestConfig']); + const result = parseOnlineEvalOutputs({}, [{ name: 'TestConfig' }]); expect(result).toEqual({}); }); }); diff --git a/src/cli/cloudformation/__tests__/outputs.test.ts b/src/cli/cloudformation/__tests__/outputs.test.ts index d12ddb689..24f39b451 100644 --- a/src/cli/cloudformation/__tests__/outputs.test.ts +++ b/src/cli/cloudformation/__tests__/outputs.test.ts @@ -469,3 +469,117 @@ describe('buildDeployedState with policy data', () => { expect(result.targets.default!.resources?.policyEngines).toBeUndefined(); }); }); + +describe('buildDeployedState carry-forward', () => { + it('carries forward abTests from existing state', () => { + const existingState = { + targets: { + default: { + resources: { + stackName: 'TestStack', + abTests: { + TestExperiment: { + abTestId: 'abt-123', + abTestArn: 'arn:aws:bedrock:us-east-1:123456789012:ab-test/abt-123', + }, + }, + }, + }, + }, + }; + + const result = buildDeployedState({ + targetName: 'default', + stackName: 'TestStack', + agents: {}, + gateways: {}, + existingState, + }); + + expect(result.targets.default!.resources?.abTests).toEqual({ + TestExperiment: { + abTestId: 'abt-123', + abTestArn: 'arn:aws:bedrock:us-east-1:123456789012:ab-test/abt-123', + }, + }); + }); + + it('carries forward httpGateways from existing state', () => { + const existingState = { + targets: { + default: { + resources: { + stackName: 'TestStack', + httpGateways: { + MyHttpGw: { + gatewayId: 'hgw-456', + gatewayArn: 'arn:aws:bedrock:us-east-1:123456789012:http-gateway/hgw-456', + }, + }, + }, + }, + }, + }; + + const result = buildDeployedState({ + targetName: 'default', + stackName: 'TestStack', + agents: {}, + gateways: {}, + existingState, + }); + + expect(result.targets.default!.resources?.httpGateways).toEqual({ + MyHttpGw: { + gatewayId: 'hgw-456', + gatewayArn: 'arn:aws:bedrock:us-east-1:123456789012:http-gateway/hgw-456', + }, + }); + }); + + it('does not carry forward empty abTests', () => { + const existingState = { + targets: { + default: { + resources: { + stackName: 'TestStack', + abTests: {}, + }, + }, + }, + }; + + const result = buildDeployedState({ + targetName: 'default', + stackName: 'TestStack', + agents: {}, + gateways: {}, + existingState, + }); + + expect(result.targets.default!.resources?.abTests).toBeUndefined(); + }); + + it('does not carry forward empty httpGateways', () => { + const existingState = { + targets: { + default: { + resources: { + stackName: 'TestStack', + httpGateways: {}, + }, + }, + }, + }; + + const result = buildDeployedState({ + targetName: 'default', + stackName: 'TestStack', + agents: {}, + gateways: {}, + existingState, + }); + + expect(result.targets.default!.resources?.httpGateways).toBeUndefined(); + }); +}); diff --git a/src/cli/cloudformation/outputs.ts b/src/cli/cloudformation/outputs.ts index 28ced03c1..5f574aefe 100644 --- a/src/cli/cloudformation/outputs.ts +++ b/src/cli/cloudformation/outputs.ts @@ -2,6 +2,7 @@ import type { AgentCoreDeployedState, DeployedState, EvaluatorDeployedState, + HarnessDeployedState, MemoryDeployedState, OnlineEvalDeployedState, PolicyDeployedState, @@ -250,13 +251,13 @@ export function parseEvaluatorOutputs( */ export function parseOnlineEvalOutputs( outputs: StackOutputs, - onlineEvalNames: string[] + onlineEvalSpecs: { name: string; agent?: string; endpoint?: string }[] ): Record { const configs: Record = {}; const outputKeys = Object.keys(outputs); - for (const configName of onlineEvalNames) { - const pascal = toPascalId('OnlineEval', configName); + for (const spec of onlineEvalSpecs) { + const pascal = toPascalId('OnlineEval', spec.name); const idPrefix = `Application${pascal}IdOutput`; const arnPrefix = `Application${pascal}ArnOutput`; @@ -264,9 +265,11 @@ export function parseOnlineEvalOutputs( const arnKey = outputKeys.find(k => k.startsWith(arnPrefix)); if (idKey && arnKey) { - configs[configName] = { + configs[spec.name] = { onlineEvaluationConfigId: outputs[idKey]!, onlineEvaluationConfigArn: outputs[arnKey]!, + ...(spec.agent && { agent: spec.agent }), + ...(spec.endpoint && { endpoint: spec.endpoint }), }; } } @@ -386,6 +389,7 @@ export interface BuildDeployedStateOptions { onlineEvalConfigs?: Record; policyEngines?: Record; policies?: Record; + harnesses?: Record; runtimeEndpoints?: Record; } @@ -406,6 +410,7 @@ export function buildDeployedState(opts: BuildDeployedStateOptions): DeployedSta onlineEvalConfigs, policyEngines, policies, + harnesses, runtimeEndpoints, } = opts; const targetState: TargetDeployedState = { @@ -441,11 +446,34 @@ export function buildDeployedState(opts: BuildDeployedStateOptions): DeployedSta targetState.resources!.onlineEvalConfigs = onlineEvalConfigs; } + // Add harness state if harnesses exist + if (harnesses && Object.keys(harnesses).length > 0) { + targetState.resources!.harnesses = harnesses; + } + // Add runtime endpoint state if endpoints exist if (runtimeEndpoints && Object.keys(runtimeEndpoints).length > 0) { targetState.resources!.runtimeEndpoints = runtimeEndpoints; } + // Carry forward config bundles from existing state (managed post-deploy, not via CFN outputs) + const existingConfigBundles = existingState?.targets?.[targetName]?.resources?.configBundles; + if (existingConfigBundles && Object.keys(existingConfigBundles).length > 0) { + targetState.resources!.configBundles = existingConfigBundles; + } + + // Carry forward AB tests from existing state (managed post-deploy, not via CFN outputs) + const existingABTests = existingState?.targets?.[targetName]?.resources?.abTests; + if (existingABTests && Object.keys(existingABTests).length > 0) { + targetState.resources!.abTests = existingABTests; + } + + // Carry forward HTTP gateways from existing state (managed post-deploy, not via CFN outputs) + const existingHttpGateways = existingState?.targets?.[targetName]?.resources?.httpGateways; + if (existingHttpGateways && Object.keys(existingHttpGateways).length > 0) { + targetState.resources!.httpGateways = existingHttpGateways; + } + return { targets: { ...existingState?.targets, diff --git a/src/cli/commands/abtest/command.ts b/src/cli/commands/abtest/command.ts new file mode 100644 index 000000000..cc236cdb3 --- /dev/null +++ b/src/cli/commands/abtest/command.ts @@ -0,0 +1,199 @@ +/** + * AB Test commands. + * + * `agentcore ab-test ` — fetches and displays full AB test details + * from the data plane API, including evaluation scores/metrics. + */ +import { ConfigIO } from '../../../lib'; +import { getABTest, listABTests } from '../../aws/agentcore-ab-tests'; +import type { GetABTestResult } from '../../aws/agentcore-ab-tests'; +import { dnsSuffix } from '../../aws/partition'; +import { getErrorMessage } from '../../errors'; +import type { Command } from '@commander-js/extra-typings'; + +// ============================================================================ +// Helpers +// ============================================================================ + +async function getRegion(cliRegion?: string): Promise { + if (cliRegion) return cliRegion; + try { + const configIO = new ConfigIO(); + const targets = await configIO.resolveAWSDeploymentTargets(); + if (targets.length > 0) return targets[0]!.region; + } catch { + // Fall through to env vars + } + return process.env.AWS_DEFAULT_REGION ?? process.env.AWS_REGION ?? 'us-east-1'; +} + +async function resolveABTestId( + testName: string, + region: string +): Promise<{ abTestId: string; region: string; error?: string }> { + let projectName: string | undefined; + try { + const configIO = new ConfigIO(); + const deployedState = await configIO.readDeployedState(); + const awsTargets = await configIO.readAWSDeploymentTargets(); + + try { + const projectSpec = await configIO.readProjectSpec(); + projectName = projectSpec.name; + } catch { + // Project spec unavailable + } + + for (const [targetName, target] of Object.entries(deployedState.targets ?? {})) { + const abTests = target.resources?.abTests; + if (abTests?.[testName]) { + const targetConfig = awsTargets.find(t => t.name === targetName); + const resolvedRegion = targetConfig?.region ?? region; + return { abTestId: abTests[testName].abTestId, region: resolvedRegion }; + } + } + } catch { + // No deployed state available + } + + try { + const result = await listABTests({ region, maxResults: 100 }); + // Match against both prefixed name ({projectName}_{testName}) and bare testName (backwards compat) + const prefixedName = projectName ? `${projectName}_${testName}` : undefined; + // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- boolean OR, not nullish coalescing + const match = result.abTests.find(t => (prefixedName && t.name === prefixedName) || t.name === testName); + if (match) { + return { abTestId: match.abTestId, region }; + } + } catch { + // API call failed + } + + return { abTestId: '', region, error: `AB test "${testName}" not found in deployed state or API.` }; +} + +function gatewayUrlFromArn(arn: string): string { + const parts = arn.split(':'); + const region = parts[3]; + const gatewayId = parts[5]?.split('/')[1]; + if (region && gatewayId) { + return `https://${gatewayId}.gateway.bedrock-agentcore.${region}.${dnsSuffix(region)}`; + } + return arn; +} + +function formatABTestDetails(test: GetABTestResult): string { + const lines: string[] = []; + lines.push(`AB Test: ${test.name}`); + lines.push(` Status: ${test.status}`); + lines.push(` Execution: ${test.executionStatus}`); + lines.push(` Invocation URL: ${gatewayUrlFromArn(test.gatewayArn)}//invocations`); + lines.push( + ` Online Eval: ${'onlineEvaluationConfigArn' in test.evaluationConfig ? test.evaluationConfig.onlineEvaluationConfigArn : 'per-variant'}` + ); + if (test.description) lines.push(` Description: ${test.description}`); + + for (const variant of test.variants) { + const bundleRef = variant.variantConfiguration.configurationBundle; + const targetRef = variant.variantConfiguration.target; + if (targetRef) { + lines.push(` Variant ${variant.name}: weight=${variant.weight}, target=${targetRef.name}`); + } else if (bundleRef) { + lines.push( + ` Variant ${variant.name}: weight=${variant.weight}, bundle=${bundleRef.bundleArn}, version=${bundleRef.bundleVersion}` + ); + } + } + + // TODO(post-preview): Re-enable max duration display once configurable duration is launched. + // if (test.maxDurationDays) lines.push(` Max Duration: ${test.maxDurationDays} days`); + if (test.startedAt) lines.push(` Started: ${test.startedAt}`); + if (test.stoppedAt) lines.push(` Stopped: ${test.stoppedAt}`); + if (test.failureReason) lines.push(` Failure: ${test.failureReason}`); + + if (test.results) { + lines.push(' Results:'); + if (test.results.analysisTimestamp) { + lines.push(` Analysis Time: ${test.results.analysisTimestamp}`); + } + for (const metric of test.results.evaluatorMetrics) { + lines.push(` Evaluator: ${metric.evaluatorArn}`); + lines.push( + ` Control: samples=${metric.controlStats.sampleSize}, mean=${metric.controlStats.mean.toFixed(4)}` + ); + for (const vr of metric.variantResults) { + lines.push( + ` ${vr.treatmentName}: samples=${vr.sampleSize}, mean=${vr.mean.toFixed(4)}, significant=${vr.isSignificant}` + ); + if (vr.absoluteChange !== undefined) + lines.push(` Change: ${vr.absoluteChange.toFixed(4)} (${(vr.percentChange ?? 0).toFixed(2)}%)`); + if (vr.pValue !== undefined) lines.push(` p-value: ${vr.pValue.toFixed(6)}`); + if (vr.confidenceInterval) { + lines.push( + ` CI: [${vr.confidenceInterval.lower?.toFixed(4)}, ${vr.confidenceInterval.upper?.toFixed(4)}]` + ); + } + } + } + } + + return lines.join('\n'); +} + +// ============================================================================ +// Command registration +// ============================================================================ + +export function registerABTestCommand(program: Command): void { + program + .command('ab-test') + .description('[preview] View A/B test details and results') + .argument('', 'AB test name') + .option('--region ', 'AWS region') + .option('--json', 'Output as JSON') + .action(async (name: string, cliOptions: { region?: string; json?: boolean }) => { + try { + const region = await getRegion(cliOptions.region); + const { abTestId, error } = await resolveABTestId(name, region); + if (error) { + if (cliOptions.json) { + console.log(JSON.stringify({ success: false, error })); + } else { + console.error(error); + } + process.exit(1); + } + const result = await getABTest({ region, abTestId }); + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + process.exit(0); + } else if (process.stdout.isTTY) { + // Render TUI detail screen with key bindings + const [{ render }, { default: React }, { ABTestDetailScreen }] = await Promise.all([ + import('ink'), + import('react'), + import('../../tui/screens/ab-test'), + ]); + render( + React.createElement(ABTestDetailScreen, { + abTestId, + region, + onExit: () => process.exit(0), + }) + ); + return; + } else { + console.log(formatABTestDetails(result)); + process.exit(0); + } + } catch (error) { + if (cliOptions.json) { + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + } else { + console.error(`Error: ${getErrorMessage(error)}`); + } + process.exit(1); + } + }); +} diff --git a/src/cli/commands/abtest/index.ts b/src/cli/commands/abtest/index.ts new file mode 100644 index 000000000..0ff25efc5 --- /dev/null +++ b/src/cli/commands/abtest/index.ts @@ -0,0 +1 @@ +export { registerABTestCommand } from './command'; diff --git a/src/cli/commands/add/__tests__/validate-harness.test.ts b/src/cli/commands/add/__tests__/validate-harness.test.ts new file mode 100644 index 000000000..6309ab623 --- /dev/null +++ b/src/cli/commands/add/__tests__/validate-harness.test.ts @@ -0,0 +1,97 @@ +import type { AddHarnessCliOptions } from '../types'; +import { validateAddHarnessOptions } from '../validate'; +import { describe, expect, it } from 'vitest'; + +describe('validateAddHarnessOptions', () => { + it('returns valid for no auth options', () => { + const options: AddHarnessCliOptions = {}; + expect(validateAddHarnessOptions(options)).toEqual({ valid: true }); + }); + + it('returns valid for AWS_IAM', () => { + const options: AddHarnessCliOptions = { authorizerType: 'AWS_IAM' }; + expect(validateAddHarnessOptions(options)).toEqual({ valid: true }); + }); + + it('returns valid for CUSTOM_JWT with all required fields', () => { + const options: AddHarnessCliOptions = { + authorizerType: 'CUSTOM_JWT', + discoveryUrl: 'https://cognito-idp.us-east-1.amazonaws.com/us-east-1_abc123/.well-known/openid-configuration', + allowedAudience: 'aud1,aud2', + }; + expect(validateAddHarnessOptions(options)).toEqual({ valid: true }); + }); + + it('rejects invalid authorizer type', () => { + const options: AddHarnessCliOptions = { authorizerType: 'INVALID' as any }; + const result = validateAddHarnessOptions(options); + expect(result.valid).toBe(false); + expect(result.error).toContain('Invalid authorizer type'); + }); + + it('rejects CUSTOM_JWT without discoveryUrl', () => { + const options: AddHarnessCliOptions = { authorizerType: 'CUSTOM_JWT' }; + const result = validateAddHarnessOptions(options); + expect(result.valid).toBe(false); + expect(result.error).toContain('--discovery-url is required'); + }); + + it('rejects clientId without CUSTOM_JWT', () => { + const options: AddHarnessCliOptions = { clientId: 'abc' }; + const result = validateAddHarnessOptions(options); + expect(result.valid).toBe(false); + expect(result.error).toContain('OAuth client credentials are only valid with CUSTOM_JWT authorizer'); + }); + + it('rejects unknown tool name', () => { + const result = validateAddHarnessOptions({ tools: 'agentcore_browser,foo_tool' }); + expect(result.valid).toBe(false); + expect(result.error).toContain("Unknown tool 'foo_tool'"); + }); + + it('rejects remote_mcp without --mcp-name', () => { + const result = validateAddHarnessOptions({ tools: 'remote_mcp', mcpUrl: 'https://example.com' }); + expect(result.valid).toBe(false); + expect(result.error).toContain('--mcp-name is required'); + }); + + it('rejects remote_mcp without --mcp-url', () => { + const result = validateAddHarnessOptions({ tools: 'remote_mcp', mcpName: 'my-mcp' }); + expect(result.valid).toBe(false); + expect(result.error).toContain('--mcp-url is required'); + }); + + it('rejects agentcore_gateway without --gateway-arn', () => { + const result = validateAddHarnessOptions({ tools: 'agentcore_gateway' }); + expect(result.valid).toBe(false); + expect(result.error).toContain('--gateway-arn is required'); + }); + + it('rejects invalid --gateway-outbound-auth value', () => { + const result = validateAddHarnessOptions({ gatewayOutboundAuth: 'iam' }); + expect(result.valid).toBe(false); + expect(result.error).toContain("Invalid --gateway-outbound-auth 'iam'"); + }); + + it('rejects oauth gateway auth without --gateway-provider-arn', () => { + const result = validateAddHarnessOptions({ gatewayOutboundAuth: 'oauth', gatewayScopes: 'read' }); + expect(result.valid).toBe(false); + expect(result.error).toContain('--gateway-provider-arn is required'); + }); + + it('rejects oauth gateway auth without --gateway-scopes', () => { + const result = validateAddHarnessOptions({ gatewayOutboundAuth: 'oauth', gatewayProviderArn: 'arn:aws:...' }); + expect(result.valid).toBe(false); + expect(result.error).toContain('--gateway-scopes is required'); + }); + + it('accepts valid tools with required companion flags', () => { + const result = validateAddHarnessOptions({ + tools: 'agentcore_browser,remote_mcp,agentcore_gateway', + mcpName: 'my-mcp', + mcpUrl: 'https://mcp.example.com', + gatewayArn: 'arn:aws:bedrock:us-east-1:123456789012:gateway/gw', + }); + expect(result).toEqual({ valid: true }); + }); +}); diff --git a/src/cli/commands/add/tool-action.ts b/src/cli/commands/add/tool-action.ts new file mode 100644 index 000000000..cced82d81 --- /dev/null +++ b/src/cli/commands/add/tool-action.ts @@ -0,0 +1,177 @@ +import { ConfigIO } from '../../../lib'; +import type { HarnessGatewayOutboundAuth, HarnessSpec } from '../../../schema'; +import type { HarnessToolType } from '../../../schema/schemas/primitives/harness'; + +export interface AddToolOptions { + harness: string; + type: string; + name: string; + url?: string; + browserArn?: string; + codeInterpreterArn?: string; + gatewayArn?: string; + gateway?: string; + outboundAuth?: string; + providerArn?: string; + scopes?: string; + grantType?: string; + json?: boolean; +} + +const VALID_OUTBOUND_AUTH_TYPES = ['awsIam', 'none', 'oauth'] as const; +const VALID_GRANT_TYPES = ['CLIENT_CREDENTIALS', 'USER_FEDERATION'] as const; +const ARN_PATTERN = /^arn:[^:]+:/; + +export interface AddToolResult { + success: boolean; + error?: string; + harnessName?: string; + toolName?: string; +} + +const VALID_TOOL_TYPES: HarnessToolType[] = [ + 'agentcore_browser', + 'agentcore_code_interpreter', + 'remote_mcp', + 'agentcore_gateway', + 'inline_function', +]; + +export async function handleAddTool(options: AddToolOptions): Promise { + const { harness, type, name } = options; + + if (!VALID_TOOL_TYPES.includes(type as HarnessToolType)) { + return { + success: false, + error: `Invalid tool type '${type}'. Valid types: ${VALID_TOOL_TYPES.join(', ')}`, + }; + } + + const toolType = type as HarnessToolType; + + if (toolType === 'remote_mcp' && !options.url) { + return { success: false, error: '--url is required for remote_mcp tools' }; + } + + if (toolType === 'agentcore_gateway' && !options.gatewayArn && !options.gateway) { + return { success: false, error: '--gateway-arn or --gateway is required for agentcore_gateway tools' }; + } + + let outboundAuth: HarnessGatewayOutboundAuth | undefined; + if (options.outboundAuth !== undefined) { + if (toolType !== 'agentcore_gateway') { + return { success: false, error: '--outbound-auth is only valid for agentcore_gateway tools' }; + } + if (!VALID_OUTBOUND_AUTH_TYPES.includes(options.outboundAuth as (typeof VALID_OUTBOUND_AUTH_TYPES)[number])) { + return { + success: false, + error: `Invalid --outbound-auth '${options.outboundAuth}'. Valid: ${VALID_OUTBOUND_AUTH_TYPES.join(', ')}`, + }; + } + if (options.outboundAuth === 'awsIam' || options.outboundAuth === 'none') { + if (options.providerArn || options.scopes || options.grantType) { + return { + success: false, + error: '--provider-arn, --scopes, and --grant-type are only valid with --outbound-auth oauth', + }; + } + outboundAuth = options.outboundAuth === 'awsIam' ? { awsIam: {} } : { none: {} }; + } else { + if (!options.providerArn) { + return { success: false, error: '--provider-arn is required when --outbound-auth oauth' }; + } + if (!ARN_PATTERN.test(options.providerArn)) { + return { success: false, error: `Invalid --provider-arn '${options.providerArn}': must be a valid ARN` }; + } + if (!options.scopes) { + return { success: false, error: '--scopes is required when --outbound-auth oauth' }; + } + const scopes = options.scopes + .split(',') + .map(s => s.trim()) + .filter(Boolean); + if (scopes.length === 0) { + return { success: false, error: '--scopes must contain at least one scope' }; + } + if ( + options.grantType !== undefined && + !VALID_GRANT_TYPES.includes(options.grantType as (typeof VALID_GRANT_TYPES)[number]) + ) { + return { + success: false, + error: `Invalid --grant-type '${options.grantType}'. Valid: ${VALID_GRANT_TYPES.join(', ')}`, + }; + } + outboundAuth = { + oauth: { + providerArn: options.providerArn, + scopes, + ...(options.grantType && { grantType: options.grantType as (typeof VALID_GRANT_TYPES)[number] }), + }, + }; + } + } + + const configIO = new ConfigIO(); + + // Resolve --gateway (project name) to ARN from deployed-state + let resolvedGatewayArn = options.gatewayArn; + if (toolType === 'agentcore_gateway' && options.gateway && !resolvedGatewayArn) { + try { + const deployedState = await configIO.readDeployedState(); + const targetNames = Object.keys(deployedState.targets); + if (targetNames.length === 0) { + return { success: false, error: 'No deployed targets found. Deploy the gateway first.' }; + } + const targetState = deployedState.targets[targetNames[0]!]; + const gatewayState = targetState?.resources?.mcp?.gateways?.[options.gateway]; + if (!gatewayState) { + return { + success: false, + error: `Gateway '${options.gateway}' not found in deployed state. Deploy it first or use --gateway-arn.`, + }; + } + resolvedGatewayArn = gatewayState.gatewayArn; + } catch { + return { success: false, error: 'Could not read deployed state. Deploy the gateway first or use --gateway-arn.' }; + } + } + + let harnessSpec: HarnessSpec; + try { + harnessSpec = await configIO.readHarnessSpec(harness); + } catch { + return { + success: false, + error: `Harness '${harness}' not found. Check the name or run 'agentcore add harness' first.`, + }; + } + + const existingTool = harnessSpec.tools.find(t => t.name === name); + if (existingTool) { + return { success: false, error: `Tool '${name}' already exists in harness '${harness}'` }; + } + + const toolEntry: HarnessSpec['tools'][number] = { type: toolType, name }; + + if (toolType === 'remote_mcp') { + toolEntry.config = { remoteMcp: { url: options.url! } }; + } else if (toolType === 'agentcore_browser' && options.browserArn) { + toolEntry.config = { agentCoreBrowser: { browserArn: options.browserArn } }; + } else if (toolType === 'agentcore_code_interpreter' && options.codeInterpreterArn) { + toolEntry.config = { agentCoreCodeInterpreter: { codeInterpreterArn: options.codeInterpreterArn } }; + } else if (toolType === 'agentcore_gateway') { + toolEntry.config = { + agentCoreGateway: { + gatewayArn: resolvedGatewayArn!, + ...(outboundAuth && { outboundAuth }), + }, + }; + } + + harnessSpec.tools.push(toolEntry); + + await configIO.writeHarnessSpec(harness, harnessSpec); + + return { success: true, harnessName: harness, toolName: name }; +} diff --git a/src/cli/commands/add/tool-command.ts b/src/cli/commands/add/tool-command.ts new file mode 100644 index 000000000..252c6a286 --- /dev/null +++ b/src/cli/commands/add/tool-command.ts @@ -0,0 +1,82 @@ +import { findConfigRoot } from '../../../lib'; +import { getErrorMessage } from '../../errors'; +import { handleAddTool } from './tool-action'; +import type { Command } from '@commander-js/extra-typings'; + +export function registerAddTool(addCmd: Command): void { + addCmd + .command('tool') + .description('Add a tool to a harness') + .requiredOption('--harness ', 'Target harness name') + .requiredOption( + '--type ', + 'Tool type: agentcore_browser, agentcore_code_interpreter, remote_mcp, agentcore_gateway, inline_function' + ) + .requiredOption('--name ', 'Tool name') + .option('--url ', 'MCP server URL (required for remote_mcp)') + .option('--browser-arn ', 'Custom browser ARN (optional for agentcore_browser)') + .option('--code-interpreter-arn ', 'Custom code interpreter ARN (optional for agentcore_code_interpreter)') + .option('--gateway-arn ', 'Gateway ARN (for agentcore_gateway)') + .option('--gateway ', 'Project gateway name — resolves ARN from deployed state (for agentcore_gateway)') + .option( + '--outbound-auth ', + 'Gateway outbound auth: awsIam, none, or oauth (default: awsIam if omitted) [agentcore_gateway]' + ) + .option('--provider-arn ', 'OAuth credential provider ARN (required when --outbound-auth oauth)') + .option( + '--scopes ', + 'Comma-separated OAuth scopes (required when --outbound-auth oauth), e.g. "openid,profile" or "https://api.example.com/read"' + ) + .option( + '--grant-type ', + 'OAuth grant type: CLIENT_CREDENTIALS or USER_FEDERATION (for --outbound-auth oauth)' + ) + .option('--json', 'Output as JSON') + .action(async cliOptions => { + if (!findConfigRoot()) { + console.error('No agentcore project found. Run `agentcore create` first.'); + process.exit(1); + } + + try { + const result = await handleAddTool({ + harness: cliOptions.harness, + type: cliOptions.type, + name: cliOptions.name, + url: cliOptions.url, + browserArn: cliOptions.browserArn, + codeInterpreterArn: cliOptions.codeInterpreterArn, + gatewayArn: cliOptions.gatewayArn, + gateway: cliOptions.gateway, + outboundAuth: cliOptions.outboundAuth, + providerArn: cliOptions.providerArn, + scopes: cliOptions.scopes, + grantType: cliOptions.grantType, + json: cliOptions.json, + }); + + if (!result.success) { + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.error(result.error); + } + process.exit(1); + } + + if (cliOptions.json) { + console.log(JSON.stringify(result)); + } else { + console.log(`Added tool '${result.toolName}' to harness '${result.harnessName}'.`); + console.log(`Run 'agentcore deploy' to apply changes.`); + } + } catch (error) { + if (cliOptions.json) { + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + } else { + console.error(getErrorMessage(error)); + } + process.exit(1); + } + }); +} diff --git a/src/cli/commands/add/types.ts b/src/cli/commands/add/types.ts index ad3b531b4..47e7c0ad9 100644 --- a/src/cli/commands/add/types.ts +++ b/src/cli/commands/add/types.ts @@ -37,6 +37,7 @@ export interface AddAgentOptions extends VpcOptions { idleTimeout?: number | string; maxLifetime?: number | string; sessionStorageMountPath?: string; + withConfigBundle?: boolean; json?: boolean; } @@ -106,6 +107,44 @@ export interface AddGatewayTargetResult { error?: string; } +// Harness types +export interface AddHarnessCliOptions { + name?: string; + modelProvider?: string; + modelId?: string; + apiKeyArn?: string; + container?: string; + memory?: boolean; + maxIterations?: number; + maxTokens?: number; + timeout?: number; + truncationStrategy?: string; + networkMode?: string; + subnets?: string; + securityGroups?: string; + idleTimeout?: number; + maxLifetime?: number; + sessionStorage?: string; + withInvokeScript?: boolean; + systemPrompt?: string; + tools?: string; + mcpName?: string; + mcpUrl?: string; + gatewayArn?: string; + gatewayOutboundAuth?: string; + gatewayProviderArn?: string; + gatewayScopes?: string; + authorizerType?: RuntimeAuthorizerType; + discoveryUrl?: string; + allowedAudience?: string; + allowedClients?: string; + allowedScopes?: string; + customClaims?: string; + clientId?: string; + clientSecret?: string; + json?: boolean; +} + // Memory types (v2: no owner/user concept) export interface AddMemoryOptions { name?: string; diff --git a/src/cli/commands/add/validate.ts b/src/cli/commands/add/validate.ts index 15ec081ab..32f858c0f 100644 --- a/src/cli/commands/add/validate.ts +++ b/src/cli/commands/add/validate.ts @@ -27,6 +27,7 @@ import type { AddCredentialOptions, AddGatewayOptions, AddGatewayTargetOptions, + AddHarnessCliOptions, AddMemoryOptions, } from './types'; import { existsSync, readFileSync } from 'fs'; @@ -791,3 +792,80 @@ export function validateAddCredentialOptions(options: AddCredentialOptions): Val return { valid: true }; } + +const VALID_HARNESS_TOOLS = [ + 'agentcore_browser', + 'agentcore_code_interpreter', + 'remote_mcp', + 'agentcore_gateway', +] as const; + +const VALID_GATEWAY_OUTBOUND_AUTH = ['awsIam', 'none', 'oauth'] as const; + +// Harness validation +export function validateAddHarnessOptions(options: AddHarnessCliOptions): ValidationResult { + if (options.tools) { + const toolNames = options.tools.split(',').map(s => s.trim()); + for (const tool of toolNames) { + if (!VALID_HARNESS_TOOLS.includes(tool as (typeof VALID_HARNESS_TOOLS)[number])) { + return { + valid: false, + error: `Unknown tool '${tool}'. Valid tools: ${VALID_HARNESS_TOOLS.join(', ')}`, + }; + } + } + + if (toolNames.includes('remote_mcp')) { + if (!options.mcpName) { + return { valid: false, error: '--mcp-name is required when --tools includes remote_mcp' }; + } + if (!options.mcpUrl) { + return { valid: false, error: '--mcp-url is required when --tools includes remote_mcp' }; + } + } + + if (toolNames.includes('agentcore_gateway')) { + if (!options.gatewayArn) { + return { valid: false, error: '--gateway-arn is required when --tools includes agentcore_gateway' }; + } + } + } + + if (options.gatewayOutboundAuth) { + if ( + !VALID_GATEWAY_OUTBOUND_AUTH.includes(options.gatewayOutboundAuth as (typeof VALID_GATEWAY_OUTBOUND_AUTH)[number]) + ) { + return { + valid: false, + error: `Invalid --gateway-outbound-auth '${options.gatewayOutboundAuth}'. Use: ${VALID_GATEWAY_OUTBOUND_AUTH.join(', ')}`, + }; + } + + if (options.gatewayOutboundAuth === 'oauth') { + if (!options.gatewayProviderArn) { + return { valid: false, error: '--gateway-provider-arn is required when --gateway-outbound-auth is oauth' }; + } + if (!options.gatewayScopes) { + return { valid: false, error: '--gateway-scopes is required when --gateway-outbound-auth is oauth' }; + } + } + } + + if (options.authorizerType) { + const authResult = RuntimeAuthorizerTypeSchema.safeParse(options.authorizerType); + if (!authResult.success) { + return { valid: false, error: 'Invalid authorizer type. Use AWS_IAM or CUSTOM_JWT' }; + } + + if (options.authorizerType === 'CUSTOM_JWT') { + const jwtResult = validateJwtAuthorizerOptions(options); + if (!jwtResult.valid) return jwtResult; + } + } + + if (options.clientId && options.authorizerType !== 'CUSTOM_JWT') { + return { valid: false, error: 'OAuth client credentials are only valid with CUSTOM_JWT authorizer' }; + } + + return { valid: true }; +} diff --git a/src/cli/commands/config-bundle/command.tsx b/src/cli/commands/config-bundle/command.tsx new file mode 100644 index 000000000..ce72f2a4b --- /dev/null +++ b/src/cli/commands/config-bundle/command.tsx @@ -0,0 +1,347 @@ +import { + getConfigurationBundleVersion, + listConfigurationBundleVersions, + updateConfigurationBundle, +} from '../../aws/agentcore-config-bundles'; +import type { + ConfigurationBundleVersionSummary, + ListConfigurationBundleVersionsFilter, +} from '../../aws/agentcore-config-bundles'; +import { getErrorMessage } from '../../errors'; +import { deepDiff } from '../../operations/config-bundle/diff-versions'; +import { resolveBundleByName } from '../../operations/config-bundle/resolve-bundle'; +import { requireProject } from '../../tui/guards'; +import type { Command } from '@commander-js/extra-typings'; +import { Box, Text, render } from 'ink'; + +// ============================================================================ +// Helpers +// ============================================================================ + +function formatTimestamp(ts: string): string { + const num = Number(ts); + if (isNaN(num)) return ts; + // API returns epoch seconds; convert to ms if needed + const ms = num < 1e12 ? num * 1000 : num; + return new Date(ms) + .toISOString() + .replace('T', ' ') + .replace(/\.\d+Z$/, 'Z'); +} + +async function resolveRegion(): Promise { + const { ConfigIO } = await import('../../../lib'); + const configIO = new ConfigIO(); + const targets = await configIO.resolveAWSDeploymentTargets(); + if (targets.length === 0) { + throw new Error('No AWS deployment targets configured. Run `agentcore deploy` first.'); + } + return targets[0]!.region; +} + +// ============================================================================ +// Version list +// ============================================================================ + +async function handleVersions(options: { + bundle: string; + branch?: string; + latestPerBranch?: boolean; + createdBy?: string; + region?: string; + json?: boolean; +}) { + const region = options.region ?? (await resolveRegion()); + const resolved = await resolveBundleByName(options.bundle, region); + + const filter: ListConfigurationBundleVersionsFilter = {}; + if (options.branch) filter.branchName = options.branch; + if (options.latestPerBranch) filter.latestPerBranch = true; + if (options.createdBy) filter.createdByName = options.createdBy; + const hasFilter = Object.keys(filter).length > 0; + + // Paginate to collect all versions + const allVersions: ConfigurationBundleVersionSummary[] = []; + let nextToken: string | undefined; + do { + const result = await listConfigurationBundleVersions({ + region, + bundleId: resolved.bundleId, + maxResults: 50, + nextToken, + ...(hasFilter && { filter }), + }); + allVersions.push(...result.versions); + nextToken = result.nextToken; + } while (nextToken); + + // Sort by creation time, newest first + allVersions.sort((a, b) => Number(b.versionCreatedAt) - Number(a.versionCreatedAt)); + + return { versions: allVersions, bundleName: options.bundle, bundleId: resolved.bundleId }; +} + +// ============================================================================ +// Diff +// ============================================================================ + +async function handleDiff(options: { bundle: string; from: string; to: string; region?: string }) { + const region = options.region ?? (await resolveRegion()); + const resolved = await resolveBundleByName(options.bundle, region); + + const [fromVersion, toVersion] = await Promise.all([ + getConfigurationBundleVersion({ region, bundleId: resolved.bundleId, versionId: options.from }), + getConfigurationBundleVersion({ region, bundleId: resolved.bundleId, versionId: options.to }), + ]); + + const diffs = deepDiff(fromVersion.components, toVersion.components); + + return { fromVersion, toVersion, diffs }; +} + +// ============================================================================ +// Command registration +// ============================================================================ + +export const registerConfigBundle = (program: Command) => { + const cmd = program + .command('config-bundle') + .alias('cb') + .description('[preview] Manage configuration bundles (use bundle name from agentcore.json, not the ID)'); + + // --- versions --- + cmd + .command('versions') + .description('List version history for a configuration bundle') + .requiredOption('--bundle ', 'Bundle name as defined in agentcore.json (e.g. "MyBundle")') + .option('--branch ', 'Filter by branch name') + .option('--latest-per-branch', 'Show only the latest version per branch') + .option('--created-by ', 'Filter by creator name (e.g. "user", "recommendation")') + .option('--region ', 'AWS region override') + .option('--json', 'Output as JSON') + .action( + async (cliOptions: { + bundle: string; + branch?: string; + latestPerBranch?: boolean; + createdBy?: string; + region?: string; + json?: boolean; + }) => { + requireProject(); + try { + const result = await handleVersions(cliOptions); + + if (cliOptions.json) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + if (result.versions.length === 0) { + render(No versions found for bundle "{cliOptions.bundle}".); + return; + } + + // Group by branch + const byBranch = new Map(); + for (const v of result.versions) { + const branch = v.lineageMetadata?.branchName ?? 'unknown'; + if (!byBranch.has(branch)) byBranch.set(branch, []); + byBranch.get(branch)!.push(v); + } + + render( + + + {result.bundleName} — {result.versions.length} version(s) + + + {[...byBranch.entries()].map(([branch, versions]) => ( + + + Branch: {branch} + + {versions.map((v, i) => { + const meta = v.lineageMetadata; + const creator = meta?.createdBy?.name ?? 'unknown'; + const message = meta?.commitMessage ?? ''; + const isLast = i === versions.length - 1; + const connector = isLast ? '└' : '├'; + return ( + + + {connector} {v.versionId}{' '} + {formatTimestamp(v.versionCreatedAt)}{' '} + {message && "{message}"} + + + {isLast ? ' ' : '│'} by: {creator} + {meta?.parentVersionIds?.length ? ( + (parent: {meta.parentVersionIds.join(', ')}) + ) : null} + + + ); + })} + + ))} + Use --json for complete output + + ); + } catch (error) { + render(Error: {getErrorMessage(error)}); + process.exit(1); + } + } + ); + + // --- diff --- + cmd + .command('diff') + .description('Diff two versions of a configuration bundle (get version IDs from `cb versions`)') + .requiredOption('--bundle ', 'Bundle name as defined in agentcore.json (e.g. "MyBundle")') + .requiredOption('--from ', 'Source version ID (from `config-bundle versions --json`)') + .requiredOption('--to ', 'Target version ID (from `config-bundle versions --json`)') + .option('--region ', 'AWS region override') + .option('--json', 'Output as JSON') + .action(async (cliOptions: { bundle: string; from: string; to: string; region?: string; json?: boolean }) => { + requireProject(); + try { + const result = await handleDiff(cliOptions); + + if (cliOptions.json) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + const fromMeta = result.fromVersion.lineageMetadata; + const toMeta = result.toVersion.lineageMetadata; + + render( + + + Diff: {result.fromVersion.versionId} → {result.toVersion.versionId} + + + From: {fromMeta?.commitMessage ?? '(no message)'} ({formatTimestamp(result.fromVersion.versionCreatedAt)}) + + + To: {toMeta?.commitMessage ?? '(no message)'} ({formatTimestamp(result.toVersion.versionCreatedAt)}) + + + {result.diffs.length === 0 ? ( + No differences found. + ) : ( + <> + {result.diffs.length} change(s): + + {result.diffs.map((d, i) => ( + + {d.path} + {d.type === 'added' && + {JSON.stringify(d.newValue)}} + {d.type === 'removed' && - {JSON.stringify(d.oldValue)}} + {d.type === 'changed' && ( + <> + - {JSON.stringify(d.oldValue)} + + {JSON.stringify(d.newValue)} + + )} + + ))} + + )} + + ); + } catch (error) { + render(Error: {getErrorMessage(error)}); + process.exit(1); + } + }); + + // --- create-branch --- + cmd + .command('create-branch') + .description('Create a new branch on an existing configuration bundle') + .requiredOption('--bundle ', 'Bundle name as defined in agentcore.json (e.g. "MyBundle")') + .requiredOption('--branch ', 'Name for the new branch') + .option('--from ', 'Parent version ID to branch from (defaults to latest version)') + .option('--commit-message ', 'Commit message for the branch point') + .option('--region ', 'AWS region override') + .option('--json', 'Output as JSON') + .action( + async (cliOptions: { + bundle: string; + branch: string; + from?: string; + commitMessage?: string; + region?: string; + json?: boolean; + }) => { + requireProject(); + try { + const region = cliOptions.region ?? (await resolveRegion()); + const resolved = await resolveBundleByName(cliOptions.bundle, region); + + // Determine parent version + let parentVersionId = cliOptions.from; + if (!parentVersionId) { + const versions = await listConfigurationBundleVersions({ + region, + bundleId: resolved.bundleId, + maxResults: 50, + }); + if (versions.versions.length === 0) { + throw new Error(`No versions found for bundle "${cliOptions.bundle}".`); + } + // Sort descending by creation time to get the latest version + const sorted = [...versions.versions].sort( + (a, b) => new Date(b.versionCreatedAt).getTime() - new Date(a.versionCreatedAt).getTime() + ); + parentVersionId = sorted[0]!.versionId; + } + + // Get the parent version's components to carry forward + const parentVersion = await getConfigurationBundleVersion({ + region, + bundleId: resolved.bundleId, + versionId: parentVersionId, + }); + + const result = await updateConfigurationBundle({ + region, + bundleId: resolved.bundleId, + components: parentVersion.components, + parentVersionIds: [parentVersionId], + branchName: cliOptions.branch, + commitMessage: cliOptions.commitMessage ?? `Create branch ${cliOptions.branch}`, + }); + + if (cliOptions.json) { + console.log(JSON.stringify(result, null, 2)); + return; + } + + render( + + + Branch "{cliOptions.branch}" created on bundle "{cliOptions.bundle}" + + + Version: {result.versionId} + + Parent: {parentVersionId} + + ); + } catch (error) { + if (cliOptions.json) { + console.log(JSON.stringify({ success: false, error: getErrorMessage(error) })); + } else { + render(Error: {getErrorMessage(error)}); + } + process.exit(1); + } + } + ); + + return cmd; +}; diff --git a/src/cli/commands/config-bundle/index.ts b/src/cli/commands/config-bundle/index.ts new file mode 100644 index 000000000..2ebcc4c68 --- /dev/null +++ b/src/cli/commands/config-bundle/index.ts @@ -0,0 +1 @@ +export { registerConfigBundle } from './command'; diff --git a/src/cli/commands/create/__tests__/create.test.ts b/src/cli/commands/create/__tests__/create.test.ts index 72ab2c64f..729cce707 100644 --- a/src/cli/commands/create/__tests__/create.test.ts +++ b/src/cli/commands/create/__tests__/create.test.ts @@ -82,7 +82,8 @@ describe('create command', () => { }); it('requires all options without --no-agent', async () => { - const result = await runCLI(['create', '--name', 'Incomplete', '--json'], testDir); + // --framework triggers the agent path, which requires --language, --model-provider, etc. + const result = await runCLI(['create', '--name', 'Incomplete', '--framework', 'Strands', '--json'], testDir); expect(result.exitCode).toBe(1); const json = JSON.parse(result.stdout); @@ -196,6 +197,44 @@ describe('create command', () => { }); }); + describe('with harness', () => { + it('uses --project-name for project and --name for harness resource', async () => { + const projectName = `HarnessProj${Date.now().toString().slice(-6)}`; + const harnessName = `HarnessResource${randomUUID().replace(/-/g, '').slice(0, 16)}`; + const result = await runCLI( + ['create', '--project-name', projectName, '--name', harnessName, '--skip-git', '--skip-install', '--json'], + testDir + ); + + expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0); + + const json = JSON.parse(result.stdout); + expect(json.success).toBe(true); + expect(json.projectPath).toMatch(new RegExp(`/${projectName}$`)); + expect(await exists(join(json.projectPath, 'app', harnessName, 'harness.json'))).toBeTruthy(); + + const projectSpec = JSON.parse(await readFile(join(json.projectPath, 'agentcore/agentcore.json'), 'utf-8')); + expect(projectSpec.name).toBe(projectName); + expect(projectSpec.harnesses[0].name).toBe(harnessName); + expect(projectSpec.harnesses[0].path).toBe(`app/${harnessName}`); + }); + + it('rejects long harness name without --project-name but accepts it with --project-name', async () => { + const harnessName = `Harness${'A'.repeat(30)}`; + const rejected = await runCLI(['create', '--name', harnessName, '--skip-install', '--json'], testDir); + expect(rejected.exitCode).toBe(1); + expect(JSON.parse(rejected.stdout).success).toBe(false); + + const projectName = `ShortProj${Date.now().toString().slice(-6)}`; + const accepted = await runCLI( + ['create', '--project-name', projectName, '--name', harnessName, '--skip-git', '--skip-install', '--json'], + testDir + ); + expect(accepted.exitCode, `stdout: ${accepted.stdout}, stderr: ${accepted.stderr}`).toBe(0); + expect(JSON.parse(accepted.stdout).success).toBe(true); + }); + }); + describe('--defaults', () => { it('creates project with defaults', async () => { const name = `Defaults${Date.now()}`; @@ -211,7 +250,11 @@ describe('create command', () => { describe('--dry-run', () => { it('shows files without creating', async () => { const name = `DryRun${Date.now()}`; - const result = await runCLI(['create', '--name', name, '--defaults', '--dry-run'], testDir); + // --framework triggers agent path where --dry-run is supported + const result = await runCLI( + ['create', '--name', name, '--defaults', '--framework', 'Strands', '--dry-run'], + testDir + ); expect(result.exitCode).toBe(0); expect(result.stdout.includes('would create') || result.stdout.includes('Dry run')).toBeTruthy(); @@ -222,7 +265,18 @@ describe('create command', () => { const projectName = `DryProj${Date.now().toString().slice(-6)}`; const agentName = `DryAgent${Date.now().toString().slice(-6)}`; const result = await runCLI( - ['create', '--project-name', projectName, '--name', agentName, '--defaults', '--dry-run', '--json'], + [ + 'create', + '--project-name', + projectName, + '--name', + agentName, + '--defaults', + '--framework', + 'Strands', + '--dry-run', + '--json', + ], testDir ); diff --git a/src/cli/commands/create/__tests__/harness-action.test.ts b/src/cli/commands/create/__tests__/harness-action.test.ts new file mode 100644 index 000000000..73d26d13a --- /dev/null +++ b/src/cli/commands/create/__tests__/harness-action.test.ts @@ -0,0 +1,126 @@ +import { exists } from '../../../../test-utils/index.js'; +import { createProjectWithHarness } from '../harness-action.js'; +import { randomUUID } from 'node:crypto'; +import { rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +describe('createProjectWithHarness', () => { + let testDir: string; + + beforeAll(() => { + testDir = join(tmpdir(), `harness-action-${randomUUID()}`); + }); + + afterAll(async () => { + await rm(testDir, { recursive: true, force: true }); + }); + + it('creates project with harness', async () => { + const name = `TestH${randomUUID().slice(0, 6)}`; + const result = await createProjectWithHarness({ + name, + cwd: testDir, + modelProvider: 'bedrock', + modelId: 'global.anthropic.claude-sonnet-4-6', + skipGit: true, + skipInstall: true, + }); + + expect(result.success, `Error: ${result.error}`).toBe(true); + expect(result.projectPath).toBeTruthy(); + + const projectPath = result.projectPath!; + const configDir = join(projectPath, 'agentcore'); + const harnessDir = join(projectPath, 'app', name); + + await expect(exists(projectPath)).resolves.toBe(true); + await expect(exists(configDir)).resolves.toBe(true); + await expect(exists(harnessDir)).resolves.toBe(true); + await expect(exists(join(harnessDir, 'harness.json'))).resolves.toBe(true); + await expect(exists(join(harnessDir, 'system-prompt.md'))).resolves.toBe(true); + }); + + it('uses projectName for project scaffold and name for harness resource', async () => { + const projectName = `Proj${randomUUID().slice(0, 6)}`; + const name = `HarnessName${randomUUID().replace(/-/g, '').slice(0, 12)}`; + const result = await createProjectWithHarness({ + name, + projectName, + cwd: testDir, + modelProvider: 'bedrock', + modelId: 'global.anthropic.claude-sonnet-4-6', + skipGit: true, + skipInstall: true, + }); + + expect(result.success, `Error: ${result.error}`).toBe(true); + expect(result.projectPath).toBe(join(testDir, projectName)); + + await expect(exists(join(result.projectPath!, 'agentcore'))).resolves.toBe(true); + await expect(exists(join(result.projectPath!, 'app', name, 'harness.json'))).resolves.toBe(true); + }); + + it('creates harness with custom options', async () => { + const name = `CustomH${randomUUID().slice(0, 6)}`; + const result = await createProjectWithHarness({ + name, + cwd: testDir, + modelProvider: 'open_ai', + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:my-key', + skipMemory: true, + maxIterations: 10, + maxTokens: 2000, + timeoutSeconds: 300, + truncationStrategy: 'sliding_window', + networkMode: 'PUBLIC', + skipGit: true, + skipInstall: true, + }); + + expect(result.success, `Error: ${result.error}`).toBe(true); + expect(result.projectPath).toBeTruthy(); + + const harnessJsonPath = join(result.projectPath!, 'app', name, 'harness.json'); + await expect(exists(harnessJsonPath)).resolves.toBe(true); + }); + + it('reports progress during creation', async () => { + const name = `ProgH${randomUUID().slice(0, 6)}`; + const progressSteps: string[] = []; + + const result = await createProjectWithHarness({ + name, + cwd: testDir, + modelProvider: 'bedrock', + modelId: 'global.anthropic.claude-sonnet-4-6', + skipGit: true, + skipInstall: true, + onProgress: (step, status) => { + if (status === 'done') { + progressSteps.push(step); + } + }, + }); + + expect(result.success, `Error: ${result.error}`).toBe(true); + expect(progressSteps).toContain('Add harness to project'); + }); + + it('handles errors gracefully', async () => { + const name = '!!!invalid-name!!!'; + const result = await createProjectWithHarness({ + name, + cwd: testDir, + modelProvider: 'bedrock', + modelId: 'global.anthropic.claude-sonnet-4-6', + skipGit: true, + skipInstall: true, + }); + + expect(result.success).toBe(false); + expect(result.error).toBeTruthy(); + }); +}); diff --git a/src/cli/commands/create/__tests__/harness-validate.test.ts b/src/cli/commands/create/__tests__/harness-validate.test.ts new file mode 100644 index 000000000..78acc0446 --- /dev/null +++ b/src/cli/commands/create/__tests__/harness-validate.test.ts @@ -0,0 +1,179 @@ +import { validateCreateHarnessOptions } from '../harness-validate.js'; +import { randomUUID } from 'node:crypto'; +import { mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +describe('validateCreateHarnessOptions', () => { + let testDir: string; + + beforeAll(() => { + testDir = join(tmpdir(), `harness-create-validate-${randomUUID()}`); + mkdirSync(testDir, { recursive: true }); + mkdirSync(join(testDir, 'existingHarness'), { recursive: true }); + mkdirSync(join(testDir, 'existingProject'), { recursive: true }); + }); + + afterAll(() => { + rmSync(testDir, { recursive: true, force: true }); + }); + + it('requires name', () => { + const result = validateCreateHarnessOptions({}, testDir); + expect(result.valid).toBe(false); + expect(result.error).toContain('--name'); + }); + + it('rejects invalid harness name starting with digit', () => { + const result = validateCreateHarnessOptions({ name: '1invalid' }, testDir); + expect(result.valid).toBe(false); + expect(result.error).toContain('letter'); + }); + + it('rejects invalid harness name with special characters', () => { + const result = validateCreateHarnessOptions({ name: 'invalid-name!' }, testDir); + expect(result.valid).toBe(false); + }); + + it('rejects existing directory', () => { + const result = validateCreateHarnessOptions({ name: 'existingHarness' }, testDir); + expect(result.valid).toBe(false); + expect(result.error).toContain('already exists'); + }); + + it('accepts valid bedrock options with defaults', () => { + const result = validateCreateHarnessOptions({ name: 'myHarness' }, testDir); + expect(result.valid).toBe(true); + }); + + it('accepts explicit model provider and id', () => { + const result = validateCreateHarnessOptions( + { + name: 'myHarness2', + modelProvider: 'bedrock', + modelId: 'us.anthropic.claude-sonnet-4-5-20250514-v1:0', + }, + testDir + ); + expect(result.valid).toBe(true); + }); + + it('requires api-key-arn for non-bedrock providers', () => { + const result = validateCreateHarnessOptions( + { + name: 'myHarness3', + modelProvider: 'open_ai', + modelId: 'gpt-4', + }, + testDir + ); + expect(result.valid).toBe(false); + expect(result.error).toContain('--api-key-arn'); + }); + + it('accepts non-bedrock provider with api-key-arn', () => { + const result = validateCreateHarnessOptions( + { + name: 'myHarness4', + modelProvider: 'open_ai', + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:my-key', + }, + testDir + ); + expect(result.valid).toBe(true); + }); + + it('normalizes titlecase model provider to lowercase', () => { + const options: any = { + name: 'myHarness5', + modelProvider: 'Bedrock', + modelId: 'test-model', + }; + const result = validateCreateHarnessOptions(options, testDir); + expect(result.valid).toBe(true); + expect(options.modelProvider).toBe('bedrock'); + }); + + it('normalizes OpenAI to open_ai', () => { + const options: any = { + name: 'myHarness6', + modelProvider: 'OpenAI', + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:my-key', + }; + const result = validateCreateHarnessOptions(options, testDir); + expect(result.valid).toBe(true); + expect(options.modelProvider).toBe('open_ai'); + }); + + it('normalizes Gemini to gemini', () => { + const options: any = { + name: 'myHarness7', + modelProvider: 'Gemini', + modelId: 'gemini-pro', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123456789012:secret:my-key', + }; + const result = validateCreateHarnessOptions(options, testDir); + expect(result.valid).toBe(true); + expect(options.modelProvider).toBe('gemini'); + }); + + it('rejects invalid model provider', () => { + const result = validateCreateHarnessOptions( + { + name: 'myHarness8', + modelProvider: 'azure', + modelId: 'test-model', + }, + testDir + ); + expect(result.valid).toBe(false); + expect(result.error).toContain('provider'); + }); + + it('applies default model provider and id', () => { + const options: any = { name: 'myHarness9' }; + const result = validateCreateHarnessOptions(options, testDir); + expect(result.valid).toBe(true); + expect(options.modelProvider).toBe('bedrock'); + expect(options.modelId).toBe('global.anthropic.claude-sonnet-4-6'); + }); + + it('accepts valid harness name with underscores when project-name is valid', () => { + const result = validateCreateHarnessOptions( + { name: 'my_valid_harness_123', projectName: 'myValidHarness123' }, + testDir + ); + expect(result.valid).toBe(true); + }); + + it('rejects harness name longer than 48 characters', () => { + const result = validateCreateHarnessOptions({ name: 'a'.repeat(49) }, testDir); + expect(result.valid).toBe(false); + }); + + it('allows long harness name when project-name is valid', () => { + const result = validateCreateHarnessOptions( + { name: `Harness${'A'.repeat(30)}`, projectName: 'ShortProject' }, + testDir + ); + expect(result.valid).toBe(true); + }); + + it('validates project-name separately from harness name', () => { + const result = validateCreateHarnessOptions( + { name: 'ValidHarness', projectName: 'ProjectNameTooLongForCli' }, + testDir + ); + expect(result.valid).toBe(false); + expect(result.error).toContain('Project name'); + }); + + it('checks folder existence using project-name', () => { + const result = validateCreateHarnessOptions({ name: 'ValidHarness2', projectName: 'existingProject' }, testDir); + expect(result.valid).toBe(false); + expect(result.error).toContain('existingProject'); + }); +}); diff --git a/src/cli/commands/create/action.ts b/src/cli/commands/create/action.ts index dbfc215d7..a00397f38 100644 --- a/src/cli/commands/create/action.ts +++ b/src/cli/commands/create/action.ts @@ -11,6 +11,7 @@ import type { import { getErrorMessage } from '../../errors'; import { checkCreateDependencies } from '../../external-requirements'; import { initGitRepo, setupPythonProject, writeEnvFile, writeGitignore } from '../../operations'; +import { createConfigBundleForAgent } from '../../operations/agent/config-bundle-defaults'; import { mapGenerateConfigToRenderConfig, mapModelProviderToIdentityProviders, @@ -131,6 +132,7 @@ export interface CreateWithAgentOptions { idleTimeout?: number; maxLifetime?: number; sessionStorageMountPath?: string; + withConfigBundle?: boolean; skipGit?: boolean; skipInstall?: boolean; skipPythonSetup?: boolean; @@ -156,6 +158,7 @@ export async function createProjectWithAgent(options: CreateWithAgentOptions): P idleTimeout, maxLifetime: maxLifetimeOpt, sessionStorageMountPath, + withConfigBundle, skipGit, skipInstall, skipPythonSetup, @@ -245,6 +248,7 @@ export async function createProjectWithAgent(options: CreateWithAgentOptions): P ...(idleTimeout !== undefined && { idleRuntimeSessionTimeout: idleTimeout }), ...(maxLifetimeOpt !== undefined && { maxLifetime: maxLifetimeOpt }), ...(sessionStorageMountPath && { sessionStorageMountPath }), + ...(withConfigBundle && { withConfigBundle }), }; // Resolve credential strategy FIRST (new project has no existing credentials) @@ -286,6 +290,11 @@ export async function createProjectWithAgent(options: CreateWithAgentOptions): P } onProgress?.('Add agent to project', 'done'); + // Auto-create config bundle when opted in + if (withConfigBundle) { + await createConfigBundleForAgent(agentName, configBaseDir); + } + // Set up Python environment if needed (unless skipped) if (language === 'Python' && !skipPythonSetup && !skipInstall) { onProgress?.('Set up Python environment', 'start'); diff --git a/src/cli/commands/create/command.tsx b/src/cli/commands/create/command.tsx index ac9d4b3ae..9be7e7bf0 100644 --- a/src/cli/commands/create/command.tsx +++ b/src/cli/commands/create/command.tsx @@ -1,6 +1,7 @@ import { getWorkingDirectory } from '../../../lib'; import type { BuildType, + HarnessModelProvider, ModelProvider, NetworkMode, ProtocolMode, @@ -9,16 +10,42 @@ import type { } from '../../../schema'; import { LIFECYCLE_TIMEOUT_MAX, LIFECYCLE_TIMEOUT_MIN } from '../../../schema'; import { getErrorMessage } from '../../errors'; +import { harnessPrimitive } from '../../primitives/registry'; import { COMMAND_DESCRIPTIONS } from '../../tui/copy'; import { requireTTY } from '../../tui/guards'; import { CreateScreen } from '../../tui/screens/create'; import { parseCommaSeparatedList } from '../shared/vpc-utils'; import { type ProgressCallback, createProject, createProjectWithAgent, getDryRunInfo } from './action'; +import { createProjectWithHarness } from './harness-action'; +import { normalizeHarnessModelProvider, validateCreateHarnessOptions } from './harness-validate'; import type { CreateOptions } from './types'; import { validateCreateOptions } from './validate'; import type { Command } from '@commander-js/extra-typings'; import { Text, render } from 'ink'; +/** Flags that trigger the agent/runtime path */ +const AGENT_PATH_FLAGS = ['framework', 'language', 'build', 'protocol', 'type', 'agentId', 'agentAliasId'] as const; + +/** Flags that are harness-only */ +const HARNESS_ONLY_FLAGS = [ + 'modelId', + 'apiKeyArn', + 'maxIterations', + 'maxTokens', + 'timeout', + 'truncationStrategy', +] as const; + +/** Determines if the agent path should be taken based on provided flags */ +function isAgentPath(options: CreateOptions): boolean { + return AGENT_PATH_FLAGS.some(flag => options[flag] !== undefined); +} + +/** Determines if any harness-only flags are present */ +function hasHarnessOnlyFlags(options: CreateOptions): boolean { + return HARNESS_ONLY_FLAGS.some(flag => options[flag] !== undefined); +} + /** Render CreateScreen for interactive TUI mode */ function handleCreateTUI(): void { const cwd = getWorkingDirectory(); @@ -73,8 +100,115 @@ function printCreateSummary( console.log(''); } -/** Handle CLI mode with progress output */ -async function handleCreateCLI(options: CreateOptions): Promise { +/** Print completion summary after successful harness create */ +function printCreateHarnessSummary(projectName: string, harnessName: string): void { + const green = '\x1b[32m'; + const cyan = '\x1b[36m'; + const dim = '\x1b[2m'; + const reset = '\x1b[0m'; + + console.log(''); + + // Created summary + console.log(`${dim}Created:${reset}`); + console.log(` ${projectName}/`); + console.log(` agentcore/ ${dim}Config and CDK project${reset}`); + console.log(` app/${harnessName}/ ${dim}Harness config${reset}`); + console.log(''); + + // Success and next steps + console.log(`${green}Harness project created successfully!${reset}`); + console.log(''); + console.log('To continue:'); + console.log(` ${cyan}cd ${projectName}${reset}`); + console.log(` ${cyan}agentcore deploy${reset}`); + console.log(''); +} + +/** Handle CLI mode for the harness path */ +async function handleCreateHarnessCLI(options: CreateOptions): Promise { + const cwd = options.outputDir ?? getWorkingDirectory(); + const name = options.name ?? options.projectName; + const projectName = options.projectName ?? name; + + const validation = validateCreateHarnessOptions( + { + name, + projectName, + modelProvider: options.modelProvider, + modelId: options.modelId, + apiKeyArn: options.apiKeyArn, + }, + cwd + ); + if (!validation.valid) { + if (options.json) { + console.log(JSON.stringify({ success: false, error: validation.error })); + } else { + console.error(validation.error); + } + process.exit(1); + } + + // Progress callback + const green = '\x1b[32m'; + const reset = '\x1b[0m'; + const onProgress: ProgressCallback | undefined = options.json + ? undefined + : (step, status) => { + if (status === 'done') console.log(`${green}[done]${reset} ${step}`); + else if (status === 'error') console.log(`\x1b[31m[error]${reset} ${step}`); + }; + + const provider = ( + options.modelProvider ? normalizeHarnessModelProvider(options.modelProvider) : 'bedrock' + ) as HarnessModelProvider; + const defaultModelIds: Record = { + bedrock: 'global.anthropic.claude-sonnet-4-6', + open_ai: 'gpt-5', + gemini: 'gemini-2.5-flash', + }; + const modelId = options.modelId ?? defaultModelIds[provider] ?? 'global.anthropic.claude-sonnet-4-6'; + + const containerOption = harnessPrimitive.parseContainerFlag(options.container); + + const result = await createProjectWithHarness({ + name: name!, + projectName: projectName!, + cwd, + modelProvider: provider, + modelId, + apiKeyArn: options.apiKeyArn, + containerUri: containerOption.containerUri, + dockerfilePath: containerOption.dockerfilePath, + skipMemory: options.harnessMemory === false, + maxIterations: options.maxIterations ? Number(options.maxIterations) : undefined, + maxTokens: options.maxTokens ? Number(options.maxTokens) : undefined, + timeoutSeconds: options.timeout ? Number(options.timeout) : undefined, + truncationStrategy: options.truncationStrategy as 'sliding_window' | 'summarization' | undefined, + networkMode: options.networkMode as NetworkMode | undefined, + subnets: parseCommaSeparatedList(options.subnets), + securityGroups: parseCommaSeparatedList(options.securityGroups), + idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, + maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, + sessionStoragePath: options.sessionStorageMountPath, + skipGit: options.skipGit, + skipInstall: options.skipInstall, + onProgress, + }); + + if (options.json) { + console.log(JSON.stringify(result)); + } else if (result.success) { + printCreateHarnessSummary(projectName!, name!); + } else { + console.error(result.error); + } + process.exit(result.success ? 0 : 1); +} + +/** Handle CLI mode with progress output for the agent/runtime path */ +async function handleCreateAgentCLI(options: CreateOptions): Promise { const cwd = options.outputDir ?? getWorkingDirectory(); const name = options.name ?? options.projectName; const projectName = options.projectName ?? name; @@ -91,7 +225,7 @@ async function handleCreateCLI(options: CreateOptions): Promise { // Handle dry-run mode if (options.dryRun) { - const result = getDryRunInfo({ name: name!, projectName, cwd, language: options.language }); + const result = getDryRunInfo({ name: name!, projectName: projectName!, cwd, language: options.language }); if (options.json) { console.log(JSON.stringify(result)); } else { @@ -131,7 +265,7 @@ async function handleCreateCLI(options: CreateOptions): Promise { }) : await createProjectWithAgent({ name: name!, - projectName, + projectName: projectName!, cwd, type: options.type as 'create' | 'import' | undefined, buildType: (options.build as BuildType) ?? 'CodeZip', @@ -150,6 +284,7 @@ async function handleCreateCLI(options: CreateOptions): Promise { idleTimeout: options.idleTimeout ? Number(options.idleTimeout) : undefined, maxLifetime: options.maxLifetime ? Number(options.maxLifetime) : undefined, sessionStorageMountPath: options.sessionStorageMountPath, + withConfigBundle: options.withConfigBundle, skipGit: options.skipGit, skipInstall: options.skipInstall, skipPythonSetup: options.skipPythonSetup, @@ -182,14 +317,14 @@ export const registerCreate = (program: Command) => { 'Project name (start with letter, alphanumeric only, max 23 chars) [non-interactive]' ) .option('--no-agent', 'Skip agent creation [non-interactive]') - .option('--defaults', 'Use defaults (Python, Strands, Bedrock, no memory) [non-interactive]') + .option('--defaults', 'Use defaults [non-interactive]') .option('--build ', 'Build type: CodeZip or Container (default: CodeZip) [non-interactive]') .option('--language ', 'Target language (default: Python) [non-interactive]') .option( '--framework ', - 'Agent framework (Strands, LangChain_LangGraph, GoogleADK, OpenAIAgents) [non-interactive]' + 'Agent framework (Strands, LangChain_LangGraph, GoogleADK, OpenAIAgents); triggers agent/runtime path [non-interactive]' ) - .option('--model-provider ', 'Model provider (Bedrock, Anthropic, OpenAI, Gemini) [non-interactive]') + .option('--model-provider ', 'Model provider: bedrock, open_ai, gemini (harness path) [non-interactive]') .option('--api-key ', 'API key for non-Bedrock providers [non-interactive]') .option('--memory