fix(e2e): separate gateway import test and detect changed tests in PR

.github/workflows/e2e-tests.yml

@@ -47,14 +47,11 @@ jobs:
     runs-on: ubuntu-latest
     environment: e2e-testing
     timeout-minutes: 30
-    strategy:
-      fail-fast: false
-      matrix:
-        cdk-source: [npm, main]
     steps:
       - uses: actions/checkout@v6
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
       - uses: actions/setup-node@v6
         with:
           node-version: '20.x'
@@ -82,7 +79,6 @@ jobs:
       # Build @aws/agentcore-cdk from source for cross-package testing.
       # Requires secrets: CDK_REPO_NAME (org/repo), CDK_REPO_TOKEN (fine-grained PAT)
       - name: Build CDK package from main
-        if: matrix.cdk-source == 'main'
         run: |
           git clone --depth 1 "https://x-access-token:${CDK_REPO_TOKEN}@github.com/${CDK_REPO}.git" /tmp/cdk-repo
           cd /tmp/cdk-repo
@@ -98,14 +94,24 @@ jobs:
       - run: npm run build
       - name: Install CLI globally
         run: npm install -g "$(npm pack | tail -1)"
-      - name: Run E2E tests (${{ matrix.cdk-source }})
+
+      - name: Detect changed e2e test files
+        id: changed
+        run: |
+          BASE_SHA=${{ github.event.pull_request.base.sha || 'HEAD~1' }}
+          CHANGED=$(git diff --name-only "$BASE_SHA"..HEAD -- 'e2e-tests/*.test.ts' \
+            | grep -v '^e2e-tests/strands-bedrock\.test\.ts$' \
+            | tr '\n' ' ')
+          echo "extra_tests=$CHANGED" >> "$GITHUB_OUTPUT"
+          echo "Changed e2e tests: ${CHANGED:-none}"
+
+      - name: Run E2E tests
         env:
           AWS_ACCOUNT_ID: ${{ steps.aws.outputs.account_id }}
           AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
           ANTHROPIC_API_KEY: ${{ env.E2E_ANTHROPIC_API_KEY }}
           OPENAI_API_KEY: ${{ env.E2E_OPENAI_API_KEY }}
           GEMINI_API_KEY: ${{ env.E2E_GEMINI_API_KEY }}
           CDK_TARBALL: ${{ env.CDK_TARBALL }}
-        # Only run Bedrock tests on PRs to avoid creating ApiKeyCredentialProviders,
-        # which have a 50-resource account limit and accumulate from interrupted runs.
-        run: npx vitest run --project e2e strands-bedrock langgraph-bedrock
+        # Always run strands-bedrock as baseline, plus any e2e test files changed in the PR
+        run: npx vitest run --project e2e e2e-tests/strands-bedrock.test.ts ${{ steps.changed.outputs.extra_tests }}

e2e-tests/import-gateway.test.ts

@@ -0,0 +1,194 @@
+import {
+  type RunResult,
+  hasAwsCredentials,
+  hasCommand,
+  parseJsonOutput,
+  prereqs,
+  spawnAndCollect,
+  stripAnsi,
+} from '../src/test-utils/index.js';
+import { installCdkTarball, runAgentCoreCLI, writeAwsTargets } from './e2e-helper.js';
+import { execSync } from 'node:child_process';
+import { randomUUID } from 'node:crypto';
+import { mkdir, readFile, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+
+const hasAws = hasAwsCredentials();
+const hasPython =
+  hasCommand('python3') &&
+  (() => {
+    try {
+      execSync('uv run --with boto3 python3 -c "import boto3"', { stdio: 'ignore' });
+      return true;
+    } catch {
+      return false;
+    }
+  })();
+const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws && hasPython;
+
+describe.sequential('e2e: import gateway', () => {
+  const region = process.env.AWS_REGION ?? 'us-east-1';
+  const fixtureDir = join(__dirname, 'fixtures', 'import');
+  const suffix = Date.now().toString().slice(-8);
+  const agentName = `E2eGw${suffix}`;
+
+  let gatewayArn: string;
+  let projectPath: string;
+  let testDir: string;
+
+  beforeAll(async () => {
+    if (!canRun) return;
+
+    const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'setup_gateway.py'], fixtureDir, {
+      AWS_REGION: region,
+    });
+    if (result.exitCode !== 0) {
+      throw new Error(
+        `setup_gateway.py failed (exit ${result.exitCode}):\nstdout: ${result.stdout}\nstderr: ${result.stderr}`
+      );
+    }
+
+    const resourcesPath = join(fixtureDir, 'bugbash-resources.json');
+    const resources = JSON.parse(await readFile(resourcesPath, 'utf-8')) as Record<string, { arn: string; id: string }>;
+    gatewayArn = resources.gateway!.arn;
+
+    testDir = join(tmpdir(), `agentcore-e2e-import-gw-${randomUUID()}`);
+    await mkdir(testDir, { recursive: true });
+
+    const createResult = await runAgentCoreCLI(
+      ['create', '--name', agentName, '--no-agent', '--defaults', '--skip-git', '--skip-python-setup', '--json'],
+      testDir
+    );
+    expect(createResult.exitCode, `Create failed: ${createResult.stderr}`).toBe(0);
+    projectPath = (parseJsonOutput(createResult.stdout) as { projectPath: string }).projectPath;
+
+    await writeAwsTargets(projectPath);
+    installCdkTarball(projectPath);
+  }, 600_000);
+
+  afterAll(async () => {
+    if (projectPath && hasAws) {
+      await runAgentCoreCLI(['remove', 'all', '--json'], projectPath);
+      const deployResult = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
+      if (deployResult.exitCode !== 0) {
+        console.warn('Teardown deploy failed:', deployResult.stderr);
+      }
+    }
+
+    try {
+      await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', 'cleanup_resources.py'], fixtureDir, {
+        AWS_REGION: region,
+      });
+    } catch {
+      /* ignore — resources may already be deleted by CFN teardown */
+    }
+
+    if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
+  }, 600_000);
+
+  const run = (args: string[]): Promise<RunResult> => runAgentCoreCLI(args, projectPath);
+
+  // ── Import test ───────────────────────────────────────────────────
+
+  it.skipIf(!canRun)(
+    'imports a gateway by ARN',
+    async () => {
+      const result = await run(['import', 'gateway', '--arn', gatewayArn]);
+
+      if (result.exitCode !== 0) {
+        console.log('Import gateway stdout:', result.stdout);
+        console.log('Import gateway stderr:', result.stderr);
+      }
+
+      expect(result.exitCode, `Import gateway failed: ${result.stderr}`).toBe(0);
+      expect(stripAnsi(result.stdout).toLowerCase()).toContain('imported successfully');
+    },
+    600_000
+  );
+
+  // ── Verification tests ────────────────────────────────────────────
+
+  it.skipIf(!canRun)(
+    'status shows imported gateway as deployed',
+    async () => {
+      const result = await run(['status', '--json']);
+
+      expect(result.exitCode, `Status failed: ${result.stderr}`).toBe(0);
+
+      const json = parseJsonOutput(result.stdout) as {
+        success: boolean;
+        resources: { resourceType: string; name: string; deploymentState: string }[];
+      };
+      expect(json.success).toBe(true);
+
+      const gateway = json.resources.find(r => r.resourceType === 'gateway');
+      expect(gateway, 'Imported gateway should appear in status').toBeDefined();
+    },
+    120_000
+  );
+
+  it.skipIf(!canRun)(
+    'agentcore.json has correct gateway fields',
+    async () => {
+      const configPath = join(projectPath, 'agentcore', 'agentcore.json');
+      const config = JSON.parse(await readFile(configPath, 'utf-8')) as {
+        agentCoreGateways: {
+          name: string;
+          resourceName?: string;
+          description?: string;
+          authorizerType: string;
+          enableSemanticSearch: boolean;
+          exceptionLevel: string;
+          executionRoleArn?: string;
+          tags?: Record<string, string>;
+          targets: { name: string; targetType: string; endpoint?: string }[];
+        }[];
+      };
+
+      expect(config.agentCoreGateways.length, 'Should have one gateway').toBe(1);
+      const gw = config.agentCoreGateways[0]!;
+
+      expect(gw.name, 'Gateway name should be set').toBeTruthy();
+      expect(gw.resourceName, 'resourceName should preserve AWS name').toBeTruthy();
+      expect(gw.description).toBe('Bugbash gateway for import testing');
+      expect(gw.authorizerType).toBe('NONE');
+      expect(gw.enableSemanticSearch).toBe(true);
+      expect(gw.exceptionLevel).toBe('DEBUG');
+      expect(gw.tags).toEqual({ env: 'bugbash', team: 'agentcore-cli' });
+
+      expect(gw.executionRoleArn, 'executionRoleArn should be preserved from AWS').toBeTruthy();
+      expect(gw.executionRoleArn).toContain('bugbash-agentcore-role');
+
+      expect(gw.targets.length, 'Should have one target').toBe(1);
+      expect(gw.targets[0]!.name).toBe('mcpTarget');
+      expect(gw.targets[0]!.targetType).toBe('mcpServer');
+      expect(gw.targets[0]!.endpoint).toBe('https://mcp.exa.ai/mcp');
+    },
+    120_000
+  );
+
+  it.skipIf(!canRun)(
+    'deployed-state.json has gateway entry',
+    async () => {
+      const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json');
+      const state = JSON.parse(await readFile(statePath, 'utf-8')) as Record<string, unknown>;
+
+      const targets = state.targets as Record<string, { resources?: { mcp?: { gateways?: Record<string, unknown> } } }>;
+      const targetEntries = Object.values(targets);
+      expect(targetEntries.length).toBeGreaterThan(0);
+
+      const firstTarget = targetEntries[0]!;
+      const gateways = firstTarget.resources?.mcp?.gateways;
+      expect(gateways, 'deployed-state should have mcp.gateways entry').toBeDefined();
+
+      const gatewayEntries = Object.values(gateways!);
+      expect(gatewayEntries.length, 'Should have one gateway in deployed state').toBe(1);
+
+      const gwState = gatewayEntries[0] as { gatewayId?: string; gatewayArn?: string };
+      expect(gwState.gatewayId, 'Gateway ID should be recorded').toBeTruthy();
+    },
+    120_000
+  );
+});

e2e-tests/import-resources.test.ts

@@ -30,7 +30,7 @@ const hasPython =
   })();
 const canRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws && hasPython;
 
-describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
+describe.sequential('e2e: import runtime/memory/evaluator', () => {
   const region = process.env.AWS_REGION ?? 'us-east-1';
   const fixtureDir = join(__dirname, 'fixtures', 'import');
   const appDir = join(fixtureDir, 'app');
@@ -40,7 +40,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
   let runtimeArn: string;
   let memoryArn: string;
   let evaluatorArn: string;
-  let gatewayArn: string;
   let projectPath: string;
   let testDir: string;
 
@@ -51,7 +50,7 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
     //    Each script creates a resource and saves its ARN/ID to bugbash-resources.json.
     //    Scripts run sequentially because save_resource() does a read-modify-write
     //    on a shared bugbash-resources.json file — parallel runs would race.
-    for (const script of ['setup_runtime_basic.py', 'setup_memory_full.py', 'setup_evaluator.py', 'setup_gateway.py']) {
+    for (const script of ['setup_runtime_basic.py', 'setup_memory_full.py', 'setup_evaluator.py']) {
       const result = await spawnAndCollect('uv', ['run', '--with', 'boto3', 'python3', script], fixtureDir, {
         AWS_REGION: region,
         DEFAULT_EVALUATOR_MODEL,
@@ -69,7 +68,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
     runtimeArn = resources['runtime-basic']!.arn;
     memoryArn = resources['memory-full']!.arn;
     evaluatorArn = resources['evaluator-llm']!.arn;
-    gatewayArn = resources.gateway!.arn;
 
     // 3. Create a destination CLI project (no agent — we'll import one)
     testDir = join(tmpdir(), `agentcore-e2e-import-${randomUUID()}`);
@@ -165,22 +163,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
     600_000
   );
 
-  it.skipIf(!canRun)(
-    'imports a gateway by ARN',
-    async () => {
-      const result = await run(['import', 'gateway', '--arn', gatewayArn]);
-
-      if (result.exitCode !== 0) {
-        console.log('Import gateway stdout:', result.stdout);
-        console.log('Import gateway stderr:', result.stderr);
-      }
-
-      expect(result.exitCode, `Import gateway failed: ${result.stderr}`).toBe(0);
-      expect(stripAnsi(result.stdout).toLowerCase()).toContain('imported successfully');
-    },
-    600_000
-  );
-
   // ── Verification tests ────────────────────────────────────────────
 
   it.skipIf(!canRun)(
@@ -205,73 +187,6 @@ describe.sequential('e2e: import runtime/memory/evaluator/gateway', () => {
 
       const evaluator = json.resources.find(r => r.resourceType === 'evaluator');
       expect(evaluator, 'Imported evaluator should appear in status').toBeDefined();
-
-      const gateway = json.resources.find(r => r.resourceType === 'gateway');
-      expect(gateway, 'Imported gateway should appear in status').toBeDefined();
-    },
-    120_000
-  );
-
-  it.skipIf(!canRun)(
-    'agentcore.json has correct gateway fields',
-    async () => {
-      const configPath = join(projectPath, 'agentcore', 'agentcore.json');
-      const config = JSON.parse(await readFile(configPath, 'utf-8')) as {
-        agentCoreGateways: {
-          name: string;
-          resourceName?: string;
-          description?: string;
-          authorizerType: string;
-          enableSemanticSearch: boolean;
-          exceptionLevel: string;
-          executionRoleArn?: string;
-          tags?: Record<string, string>;
-          targets: { name: string; targetType: string; endpoint?: string }[];
-        }[];
-      };
-
-      expect(config.agentCoreGateways.length, 'Should have one gateway').toBe(1);
-      const gw = config.agentCoreGateways[0]!;
-
-      expect(gw.name, 'Gateway name should be set').toBeTruthy();
-      expect(gw.resourceName, 'resourceName should preserve AWS name').toBeTruthy();
-      expect(gw.description).toBe('Bugbash gateway for import testing');
-      expect(gw.authorizerType).toBe('NONE');
-      expect(gw.enableSemanticSearch).toBe(true);
-      expect(gw.exceptionLevel).toBe('DEBUG');
-      expect(gw.tags).toEqual({ env: 'bugbash', team: 'agentcore-cli' });
-
-      expect(gw.executionRoleArn, 'executionRoleArn should be preserved from AWS').toBeTruthy();
-      expect(gw.executionRoleArn).toContain('bugbash-agentcore-role');
-
-      expect(gw.targets.length, 'Should have one target').toBe(1);
-      expect(gw.targets[0]!.name).toBe('mcpTarget');
-      expect(gw.targets[0]!.targetType).toBe('mcpServer');
-      expect(gw.targets[0]!.endpoint).toBe('https://mcp.exa.ai/mcp');
-    },
-    120_000
-  );
-
-  it.skipIf(!canRun)(
-    'deployed-state.json has gateway entry',
-    async () => {
-      const statePath = join(projectPath, 'agentcore', '.cli', 'deployed-state.json');
-      const state = JSON.parse(await readFile(statePath, 'utf-8')) as Record<string, unknown>;
-
-      // Gateway state is stored under targets.<targetName>.resources.mcp.gateways
-      const targets = state.targets as Record<string, { resources?: { mcp?: { gateways?: Record<string, unknown> } } }>;
-      const targetEntries = Object.values(targets);
-      expect(targetEntries.length).toBeGreaterThan(0);
-
-      const firstTarget = targetEntries[0]!;
-      const gateways = firstTarget.resources?.mcp?.gateways;
-      expect(gateways, 'deployed-state should have mcp.gateways entry').toBeDefined();
-
-      const gatewayEntries = Object.values(gateways!);
-      expect(gatewayEntries.length, 'Should have one gateway in deployed state').toBe(1);
-
-      const gwState = gatewayEntries[0] as { gatewayId?: string; gatewayArn?: string };
-      expect(gwState.gatewayId, 'Gateway ID should be recorded').toBeTruthy();
     },
     120_000
   );