diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 025b7b600..e0ee36b5b 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -47,14 +47,11 @@ jobs: runs-on: ubuntu-latest environment: e2e-testing timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - cdk-source: [npm, main] steps: - uses: actions/checkout@v6 with: ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 - uses: actions/setup-node@v6 with: node-version: '20.x' @@ -82,7 +79,6 @@ jobs: # Build @aws/agentcore-cdk from source for cross-package testing. # Requires secrets: CDK_REPO_NAME (org/repo), CDK_REPO_TOKEN (fine-grained PAT) - name: Build CDK package from main - if: matrix.cdk-source == 'main' run: | git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo cd /tmp/cdk-repo @@ -98,7 +94,18 @@ jobs: - run: npm run build - name: Install CLI globally run: npm install -g "$(npm pack | tail -1)" - - name: Run E2E tests (${{ matrix.cdk-source }}) + + - name: Detect changed e2e test files + id: changed + run: | + BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }} + CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \ + | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \ + | tr '\n' ' ') + echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT" + echo "Changed e2e tests: ${CHANGED:-none}" + + - name: Run E2E tests env: AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }} AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }} @@ -106,6 +113,5 @@ jobs: OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }} GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }} CDK_TARBALL: ${{ env.CDK_TARBALL }} - # Only run Bedrock tests on PRs to avoid creating ApiKeyCredentialProviders, - # which have a 50-resource account limit and accumulate from interrupted runs. - run: npx vitest run --project e2e strands-bedrock langgraph-bedrock + # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR + run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }} diff --git a/e2e-tests/import-gateway.test.ts b/e2e-tests/import-gateway.test.ts new file mode 100644 index 000000000..91522e592 --- /dev/null +++ b/e2e-tests/import-gateway.test.ts @@ -0,0 +1,194 @@ +import { + type RunResult, + hasAwsCredentials, + hasCommand, + parseJsonOutput, + prereqs, + spawnAndCollect, + stripAnsi, +} from '../src/test-utils/index.js'; +import { installCdkTarball, runAgentCoreCLI, writeAwsTargets } from './e2e-helper.js'; +import { execSync } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { mkdir, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; + +const hasAws = hasAwsCredentials(); +const hasPython = + hasCommand('python3') && + (() => { + try { + execSync('uv run --with boto3 python3 -c "import boto3"', { stdio: 'ignore' }); + return true; + } catch { + return false; + } + })(); +const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws && hasPython; + +describe.sequential('e2e: import gateway', () => { + const region = process.env.AWS_REGION ?? 'us-east-1'; + const fixtureDir = join(__dirname, 'fixtures', 'import'); + const suffix = Date.now().toString().slice(-8); + const agentName = `E2eGw${suffix}`; + + let gatewayArn: string; + let projectPath: string; + let testDir: string; + + beforeAll(async () => { + if (!canRun) return; + + const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'setup_gateway.py'], fixtureDir, { + AWS_REGION: region, + }); + if (result.exitCode !== 0) { + throw new Error( + `setup_gateway.py failed (exit ${result.exitCode}):\nstdout: ${result.stdout}\nstderr: ${result.stderr}` + ); + } + + const resourcesPath = join(fixtureDir, 'bugbash-resources.json'); + const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record; + gatewayArn = resources.gateway!.arn; + + testDir = join(tmpdir(), `agentcore-e2e-import-gw-${randomUUID()}`); + await mkdir(testDir, { recursive: true }); + + const createResult = await runAgentCoreCLI( + ['create', '--name', agentName, '--no-agent', '--defaults', '--skip-git', '--skip-python-setup', '--json'], + testDir + ); + expect(createResult.exitCode, `Create failed: ${createResult.stderr}`).toBe(0); + projectPath = (parseJsonOutput(createResult.stdout) as { projectPath: string }).projectPath; + + await writeAwsTargets(projectPath); + installCdkTarball(projectPath); + }, 600_000); + + afterAll(async () => { + if (projectPath && hasAws) { + await runAgentCoreCLI(['remove', 'all', '--json'], projectPath); + const deployResult = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath); + if (deployResult.exitCode !== 0) { + console.warn('Teardown deploy failed:', deployResult.stderr); + } + } + + try { + await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, { + AWS_REGION: region, + }); + } catch { + /* ignore — resources may already be deleted by CFN teardown */ + } + + if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 }); + }, 600_000); + + const run = (args: string[]): Promise => runAgentCoreCLI(args, projectPath); + + // ── Import test ─────────────────────────────────────────────────── + + it.skipIf(!canRun)( + 'imports a gateway by ARN', + async () => { + const result = await run(['import', 'gateway', '--arn', gatewayArn]); + + if (result.exitCode !== 0) { + console.log('Import gateway stdout:', result.stdout); + console.log('Import gateway stderr:', result.stderr); + } + + expect(result.exitCode, `Import gateway failed: ${result.stderr}`).toBe(0); + expect(stripAnsi(result.stdout).toLowerCase()).toContain('imported successfully'); + }, + 600_000 + ); + + // ── Verification tests ──────────────────────────────────────────── + + it.skipIf(!canRun)( + 'status shows imported gateway as deployed', + async () => { + const result = await run(['status', '--json']); + + expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0); + + const json = parseJsonOutput(result.stdout) as { + success: boolean; + resources: { resourceType: string; name: string; deploymentState: string }[]; + }; + expect(json.success).toBe(true); + + const gateway = json.resources.find(r => r.resourceType === 'gateway'); + expect(gateway, 'Imported gateway should appear in status').toBeDefined(); + }, + 120_000 + ); + + it.skipIf(!canRun)( + 'agentcore.json has correct gateway fields', + async () => { + const configPath = join(projectPath, 'agentcore', 'agentcore.json'); + const config = JSON.parse(await readFile(configPath, 'utf-8')) as { + agentCoreGateways: { + name: string; + resourceName?: string; + description?: string; + authorizerType: string; + enableSemanticSearch: boolean; + exceptionLevel: string; + executionRoleArn?: string; + tags?: Record; + targets: { name: string; targetType: string; endpoint?: string }[]; + }[]; + }; + + expect(config.agentCoreGateways.length, 'Should have one gateway').toBe(1); + const gw = config.agentCoreGateways[0]!; + + expect(gw.name, 'Gateway name should be set').toBeTruthy(); + expect(gw.resourceName, 'resourceName should preserve AWS name').toBeTruthy(); + expect(gw.description).toBe('Bugbash gateway for import testing'); + expect(gw.authorizerType).toBe('NONE'); + expect(gw.enableSemanticSearch).toBe(true); + expect(gw.exceptionLevel).toBe('DEBUG'); + expect(gw.tags).toEqual({ env: 'bugbash', team: 'agentcore-cli' }); + + expect(gw.executionRoleArn, 'executionRoleArn should be preserved from AWS').toBeTruthy(); + expect(gw.executionRoleArn).toContain('bugbash-agentcore-role'); + + expect(gw.targets.length, 'Should have one target').toBe(1); + expect(gw.targets[0]!.name).toBe('mcpTarget'); + expect(gw.targets[0]!.targetType).toBe('mcpServer'); + expect(gw.targets[0]!.endpoint).toBe('https://mcp.exa.ai/mcp'); + }, + 120_000 + ); + + it.skipIf(!canRun)( + 'deployed-state.json has gateway entry', + async () => { + const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json'); + const state = JSON.parse(await readFile(statePath, 'utf-8')) as Record; + + const targets = state.targets as Record } } }>; + const targetEntries = Object.values(targets); + expect(targetEntries.length).toBeGreaterThan(0); + + const firstTarget = targetEntries[0]!; + const gateways = firstTarget.resources?.mcp?.gateways; + expect(gateways, 'deployed-state should have mcp.gateways entry').toBeDefined(); + + const gatewayEntries = Object.values(gateways!); + expect(gatewayEntries.length, 'Should have one gateway in deployed state').toBe(1); + + const gwState = gatewayEntries[0] as { gatewayId?: string; gatewayArn?: string }; + expect(gwState.gatewayId, 'Gateway ID should be recorded').toBeTruthy(); + }, + 120_000 + ); +}); diff --git a/e2e-tests/import-resources.test.ts b/e2e-tests/import-resources.test.ts index 67733875e..e97f62f42 100644 --- a/e2e-tests/import-resources.test.ts +++ b/e2e-tests/import-resources.test.ts @@ -30,7 +30,7 @@ const hasPython = })(); const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws && hasPython; -describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { +describe.sequential('e2e: import runtime/memory/evaluator', () => { const region = process.env.AWS_REGION ?? 'us-east-1'; const fixtureDir = join(__dirname, 'fixtures', 'import'); const appDir = join(fixtureDir, 'app'); @@ -40,7 +40,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { let runtimeArn: string; let memoryArn: string; let evaluatorArn: string; - let gatewayArn: string; let projectPath: string; let testDir: string; @@ -51,7 +50,7 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { // Each script creates a resource and saves its ARN/ID to bugbash-resources.json. // Scripts run sequentially because save_resource() does a read-modify-write // on a shared bugbash-resources.json file — parallel runs would race. - for (const script of ['setup_runtime_basic.py', 'setup_memory_full.py', 'setup_evaluator.py', 'setup_gateway.py']) { + for (const script of ['setup_runtime_basic.py', 'setup_memory_full.py', 'setup_evaluator.py']) { const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', script], fixtureDir, { AWS_REGION: region, DEFAULT_EVALUATOR_MODEL, @@ -69,7 +68,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { runtimeArn = resources['runtime-basic']!.arn; memoryArn = resources['memory-full']!.arn; evaluatorArn = resources['evaluator-llm']!.arn; - gatewayArn = resources.gateway!.arn; // 3. Create a destination CLI project (no agent — we'll import one) testDir = join(tmpdir(), `agentcore-e2e-import-${randomUUID()}`); @@ -165,22 +163,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { 600_000 ); - it.skipIf(!canRun)( - 'imports a gateway by ARN', - async () => { - const result = await run(['import', 'gateway', '--arn', gatewayArn]); - - if (result.exitCode !== 0) { - console.log('Import gateway stdout:', result.stdout); - console.log('Import gateway stderr:', result.stderr); - } - - expect(result.exitCode, `Import gateway failed: ${result.stderr}`).toBe(0); - expect(stripAnsi(result.stdout).toLowerCase()).toContain('imported successfully'); - }, - 600_000 - ); - // ── Verification tests ──────────────────────────────────────────── it.skipIf(!canRun)( @@ -205,73 +187,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => { const evaluator = json.resources.find(r => r.resourceType === 'evaluator'); expect(evaluator, 'Imported evaluator should appear in status').toBeDefined(); - - const gateway = json.resources.find(r => r.resourceType === 'gateway'); - expect(gateway, 'Imported gateway should appear in status').toBeDefined(); - }, - 120_000 - ); - - it.skipIf(!canRun)( - 'agentcore.json has correct gateway fields', - async () => { - const configPath = join(projectPath, 'agentcore', 'agentcore.json'); - const config = JSON.parse(await readFile(configPath, 'utf-8')) as { - agentCoreGateways: { - name: string; - resourceName?: string; - description?: string; - authorizerType: string; - enableSemanticSearch: boolean; - exceptionLevel: string; - executionRoleArn?: string; - tags?: Record; - targets: { name: string; targetType: string; endpoint?: string }[]; - }[]; - }; - - expect(config.agentCoreGateways.length, 'Should have one gateway').toBe(1); - const gw = config.agentCoreGateways[0]!; - - expect(gw.name, 'Gateway name should be set').toBeTruthy(); - expect(gw.resourceName, 'resourceName should preserve AWS name').toBeTruthy(); - expect(gw.description).toBe('Bugbash gateway for import testing'); - expect(gw.authorizerType).toBe('NONE'); - expect(gw.enableSemanticSearch).toBe(true); - expect(gw.exceptionLevel).toBe('DEBUG'); - expect(gw.tags).toEqual({ env: 'bugbash', team: 'agentcore-cli' }); - - expect(gw.executionRoleArn, 'executionRoleArn should be preserved from AWS').toBeTruthy(); - expect(gw.executionRoleArn).toContain('bugbash-agentcore-role'); - - expect(gw.targets.length, 'Should have one target').toBe(1); - expect(gw.targets[0]!.name).toBe('mcpTarget'); - expect(gw.targets[0]!.targetType).toBe('mcpServer'); - expect(gw.targets[0]!.endpoint).toBe('https://mcp.exa.ai/mcp'); - }, - 120_000 - ); - - it.skipIf(!canRun)( - 'deployed-state.json has gateway entry', - async () => { - const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json'); - const state = JSON.parse(await readFile(statePath, 'utf-8')) as Record; - - // Gateway state is stored under targets..resources.mcp.gateways - const targets = state.targets as Record } } }>; - const targetEntries = Object.values(targets); - expect(targetEntries.length).toBeGreaterThan(0); - - const firstTarget = targetEntries[0]!; - const gateways = firstTarget.resources?.mcp?.gateways; - expect(gateways, 'deployed-state should have mcp.gateways entry').toBeDefined(); - - const gatewayEntries = Object.values(gateways!); - expect(gatewayEntries.length, 'Should have one gateway in deployed state').toBe(1); - - const gwState = gatewayEntries[0] as { gatewayId?: string; gatewayArn?: string }; - expect(gwState.gatewayId, 'Gateway ID should be recorded').toBeTruthy(); }, 120_000 );