Skip to content

Commit a1322ac

Browse files
BennettJamesBennett Sala
BennettJames
and
Bennett Sala
authored
Modifies integration tests to be runnable outside of AWS (#667)
This patch reworks the test and CI system for integration tests to make the execution of the tests more platform agnostic. A system to manage and inject credentials is added that makes the tests runnable outside of an AWS context, and substantial adaptations are made to the test setup and runtime to make them run on a pure github action or linux environment. Co-authored-by: Bennett Sala <[email protected]>
1 parent c8b003f commit a1322ac

File tree

23 files changed

+397
-204
lines changed

23 files changed

+397
-204
lines changed

‎.github/actions/integration-test/action.yaml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
name: "Run Integration Test"
2+
description: "Runs integration tests in a kinD cluster on a github action"
3+
4+
inputs:
5+
aws_role:
6+
description: "role to acquire from aws"
7+
required: true
8+
vpc_id:
9+
description: "aws vpc id to use for the test"
10+
required: true
11+
account_id:
12+
description: "aws account id to use"
13+
required: true
14+
cluster_name:
15+
description: "name of the test cluster"
16+
required: false
17+
default: "test-cluster"
18+
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Install Go 1.17
23+
uses: actions/setup-go@v3
24+
with:
25+
go-version: '1.17.*'
26+
id: go
27+
28+
- name: Setup Test Tools
29+
shell: bash
30+
run: |
31+
export GOBIN=/usr/local/bin/
32+
33+
mkdir -vp ~/.docker/cli-plugins/
34+
curl -sL -o ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/v0.3.0/buildx-v0.3.0.linux-amd64"
35+
chmod a+x ~/.docker/cli-plugins/docker-buildx
36+
37+
curl -L -o kubebuilder.tar.gz "https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.1/kubebuilder_2.3.1_linux_amd64.tar.gz"
38+
tar xzf kubebuilder.tar.gz
39+
sudo mv "kubebuilder_2.3.1_linux_amd64" /usr/local/kubebuilder
40+
export PATH=$PATH:/usr/local/kubebuilder/bin
41+
42+
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
43+
chmod 700 get_helm.sh
44+
./get_helm.sh
45+
46+
./scripts/install-controller-gen.sh
47+
./scripts/install-kubectl.sh
48+
go install sigs.k8s.io/kustomize/kustomize/v4@latest
49+
50+
go mod download
51+
go install github.com/onsi/ginkgo/[email protected]
52+
53+
- name: Run Unit Tests
54+
shell: bash
55+
run: |
56+
make test
57+
58+
- name: Configure AWS Credentials (build)
59+
uses: aws-actions/configure-aws-credentials@v1-node16
60+
with:
61+
aws-region: us-west-2
62+
role-to-assume: ${{ inputs.aws_role }}
63+
role-session-name: IntegrationTest
64+
65+
- name: Setup Kind
66+
uses: engineerd/[email protected]
67+
with:
68+
version: "v0.17.0"
69+
name: "${{ inputs.cluster_name }}"
70+
71+
- name: Run Integration Tests
72+
shell: bash
73+
env:
74+
VPC_ID: "${{ inputs.vpc_id }}"
75+
AWS_ACCOUNT_ID: "${{ inputs.account_id }}"
76+
CLUSTER_NAME: "${{ inputs.cluster_name }}"
77+
run: |
78+
KUBECONFIG="${HOME}/.kube/config" ./scripts/test-with-kind.sh

‎.github/workflows/beta-release.yaml‎

Lines changed: 39 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -7,85 +7,57 @@ on:
77
required: true
88

99
permissions:
10+
id-token: write
1011
contents: read
1112

13+
env:
14+
IMAGE_HOST: "${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com"
15+
IMAGE: "${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller"
16+
IMAGE_TAG: "${{ github.event.inputs.tag }}"
17+
IMAGE_TAG_AMD: "${{ github.event.inputs.tag }}-linux_amd64"
18+
IMAGE_TAG_ARM: "${{ github.event.inputs.tag }}-linux_arm64"
19+
1220
jobs:
13-
build:
14-
name: integration-test
15-
runs-on: [ self-hosted, aws-app-mesh-controller-for-k8s, X64 ]
21+
integration-test:
22+
name: Integration Test
23+
runs-on: ubuntu-22.04
1624
steps:
17-
- name: clean work dir from previous runs
18-
run: |
19-
rm -rf *
20-
- name: setup go 1.17
21-
uses: actions/setup-go@v3
22-
with:
23-
go-version: '1.17.*'
24-
id: go
25-
- name: setup environment
26-
run: |
27-
source ~/.bashrc
28-
- name: checkout code
29-
uses: actions/checkout@v2
25+
- name: Checkout Code
26+
uses: actions/checkout@v3
3027
with:
3128
ref: refs/tags/${{ github.event.inputs.tag }}
32-
- name: setup kind and run integration tests
33-
run: VERSION=${{ github.event.inputs.tag }} make integration-test
34-
- name: cleanup all the kind clusters
35-
run: VERSION=${{ github.event.inputs.tag }} make delete-all-kind-clusters
36-
build-arm64:
37-
name: build-arm64
38-
runs-on: [ self-hosted, aws-app-mesh-controller-for-k8s, ARM64 ]
29+
- name: Run integration test action
30+
uses: ./.github/actions/integration-test
31+
with:
32+
aws_role: "${{ secrets.BETA_TEST_AWS_ROLE }}"
33+
vpc_id: "${{ secrets.INTEG_TEST_VPC }}"
34+
account_id: "${{ secrets.BETA_AWS_ACCOUNT }}"
35+
36+
push-images:
37+
name: Build And Push Images
38+
runs-on: ubuntu-22.04
39+
needs: [ integration-test ]
3940
steps:
40-
- name: clean work dir from previous runs
41+
- name: Clean
4142
run: |
4243
rm -rf *
43-
- name: setup go 1.17
44-
uses: actions/setup-go@v3
45-
with:
46-
go-version: '1.17.*'
47-
id: go
48-
- name: setup environment
49-
run: |
50-
source ~/.bashrc
51-
- name: checkout code
52-
uses: actions/checkout@v2
44+
- name: Checkout Code
45+
uses: actions/checkout@v3
5346
with:
5447
ref: refs/tags/${{ github.event.inputs.tag }}
55-
- name: build for arm64
56-
run: |
57-
docker buildx build --platform linux/arm64 -t ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64 . --load
58-
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com
59-
docker push ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64
60-
beta-release:
61-
name: beta-release
62-
runs-on: ubuntu-18.04
63-
needs: [ build, build-arm64 ]
64-
permissions:
65-
id-token: write
66-
contents: read
67-
steps:
68-
- name: Configure AWS Credentials (build)
69-
uses: aws-actions/configure-aws-credentials@v1
70-
with:
71-
aws-region: us-west-2
72-
role-to-assume: ${{ secrets.CI_AWS_ROLE }}
73-
role-session-name: ControllerBetaRelease
74-
- name: Pull docker image
75-
run: |
76-
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com
77-
docker pull ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}
78-
docker pull ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64
79-
- name: Configure AWS Credentials (beta)
80-
uses: aws-actions/configure-aws-credentials@v1
48+
49+
- name: Configure AWS Credentials
50+
uses: aws-actions/configure-aws-credentials@v1-node16
8151
with:
8252
aws-region: us-west-2
8353
role-to-assume: ${{ secrets.BETA_AWS_ROLE }}
84-
role-session-name: ControllerBetaRelease
85-
- name: Push docker image
54+
role-session-name: ImagePusher
55+
56+
- name: Build Images
8657
run: |
87-
aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com
88-
docker tag ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }} ${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}
89-
docker tag ${{ secrets.CI_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64 ${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64
90-
docker push ${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}
91-
docker push ${{ secrets.BETA_AWS_ACCOUNT }}.dkr.ecr.us-west-2.amazonaws.com/amazon/appmesh-controller:${{ github.event.inputs.tag }}-linux_arm64
58+
aws ecr get-login-password --region us-west-2 | \
59+
docker login --username AWS --password-stdin $IMAGE_HOST
60+
# Note: right now, this pushes the amd image under the default. This
61+
# behavior should be changed to supporting multiarch shortly.
62+
docker buildx build --platform linux/amd64 -t "${IMAGE}:${IMAGE_TAG}" . --push
63+
docker buildx build --platform linux/arm64 -t "${IMAGE}:${IMAGE_TAG_ARM}" . --push

‎.github/workflows/integration-test.yaml‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -5,32 +5,19 @@ on:
55
- master
66

77
permissions:
8+
id-token: write
89
contents: read
910

1011
jobs:
11-
build:
12-
name: integration-test
13-
runs-on: [self-hosted, aws-app-mesh-controller-for-k8s, X64 ]
12+
integration-test:
13+
name: Integration Test
14+
runs-on: ubuntu-22.04
1415
steps:
15-
- name: clean work dir from previous runs
16-
run: |
17-
rm -rf *
18-
- name: setup go 1.17
19-
uses: actions/setup-go@v3
16+
- name: Checkout Code
17+
uses: actions/checkout@v3
18+
- name: Run integration test action
19+
uses: ./.github/actions/integration-test
2020
with:
21-
go-version: '1.17.*'
22-
id: go
23-
- name: setup environment
24-
run: |
25-
source ~/.bashrc
26-
- name: checkout code
27-
uses: actions/checkout@v2
28-
- name: setup kind and run integration tests
29-
run: make integration-test
30-
cleanup:
31-
runs-on: [self-hosted, aws-app-mesh-controller-for-k8s, X64]
32-
if: ${{ always() }}
33-
needs: [build]
34-
steps:
35-
- name: delete kind clusters
36-
run: make delete-all-kind-clusters
21+
aws_role: "${{ secrets.BETA_TEST_AWS_ROLE }}"
22+
vpc_id: "${{ secrets.INTEG_TEST_VPC }}"
23+
account_id: "${{ secrets.BETA_AWS_ACCOUNT }}"

‎Dockerfile‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# syntax=docker/dockerfile:experimental
22

33
# Build the controller binary
4-
FROM --platform=${TARGETPLATFORM} golang:1.17 as builder
4+
FROM --platform=${BUILDPLATFORM} golang:1.17 as builder
55

66
WORKDIR /workspace
77

@@ -15,7 +15,13 @@ ENV GOPROXY=${GOPROXY}
1515

1616
RUN go mod download
1717

18-
COPY . ./
18+
COPY ./main.go ./ATTRIBUTION.txt ./
19+
COPY .git/ ./.git/
20+
COPY pkg/ ./pkg/
21+
COPY apis/ ./apis/
22+
COPY controllers/ ./controllers/
23+
COPY mocks/ ./mocks/
24+
COPY webhooks/ ./webhooks/
1925

2026
ARG TARGETOS
2127
ARG TARGETARCH

‎config/helm/appmesh-controller/templates/deployment.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ spec:
7474
- --envoy-admin-access-enable-ipv6={{ .Values.sidecar.envoyAdminAccessEnableIPv6 }}
7575
- --dual-stack-endpoint={{ .Values.sidecar.useDualStackEndpoint }}
7676
- --fips-endpoint={{ .Values.sidecar.useFipsEndpoint }}
77+
- --envoy-aws-access-key-id={{ .Values.sidecar.envoyAwsAccessKeyId }}
78+
- --envoy-aws-secret-access-key={{ .Values.sidecar.envoyAwsSecretAccessKey }}
79+
- --envoy-aws-session-token={{ .Values.sidecar.envoyAwsSessionToken }}
7780
- --preview={{ .Values.preview }}
7881
- --enable-sds={{ .Values.sds.enabled }}
7982
- --sds-uds-path={{ .Values.sds.udsPath }}

‎pkg/inject/config.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ const (
3333
flagWaitUntilProxyReady = "wait-until-proxy-ready"
3434
flagFipsEndpoint = "fips-endpoint"
3535

36+
flagEnvoyAwsAccessKeyId = "envoy-aws-access-key-id"
37+
flagEnvoyAwsSecretAccessKey = "envoy-aws-secret-access-key"
38+
flagEnvoyAwsSessionToken = "envoy-aws-session-token"
39+
3640
flagInitImage = "init-image"
3741
flagIgnoredIPs = "ignored-ips"
3842

@@ -91,6 +95,10 @@ type Config struct {
9195
WaitUntilProxyReady bool
9296
FipsEndpoint bool
9397

98+
EnvoyAwsAccessKeyId string
99+
EnvoyAwsSecretAccessKey string
100+
EnvoyAwsSessionToken string
101+
94102
// Init container settings
95103
InitImage string
96104
IgnoredIPs string
@@ -210,6 +218,12 @@ func (cfg *Config) BindFlags(fs *pflag.FlagSet) {
210218
fs.BoolVar(&cfg.WaitUntilProxyReady, flagWaitUntilProxyReady, false,
211219
"Enable pod postStart hook to delay application startup until proxy is ready to accept traffic")
212220
fs.BoolVar(&cfg.FipsEndpoint, flagFipsEndpoint, false, "Use Fips Endpoint")
221+
fs.StringVar(&cfg.EnvoyAwsAccessKeyId, flagEnvoyAwsAccessKeyId, "",
222+
"Access key for envoy container (for integration testing)")
223+
fs.StringVar(&cfg.EnvoyAwsSecretAccessKey, flagEnvoyAwsSecretAccessKey, "",
224+
"Secret access key for envoy container (for integration testing)")
225+
fs.StringVar(&cfg.EnvoyAwsSessionToken, flagEnvoyAwsSessionToken, "",
226+
"Session token for envoy container (for integration testing)")
213227
}
214228

215229
func (cfg *Config) BindEnv() error {

‎pkg/inject/envoy.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ type envoyMutatorConfig struct {
5353
useDualStackEndpoint bool
5454
enableAdminAccessIPv6 bool
5555
useFipsEndpoint bool
56+
awsAccessKeyId string
57+
awsSecretAccessKey string
58+
awsSessionToken string
5659
}
5760

5861
func newEnvoyMutator(mutatorConfig envoyMutatorConfig, ms *appmesh.Mesh, vn *appmesh.VirtualNode) *envoyMutator {
@@ -169,6 +172,9 @@ func (m *envoyMutator) buildTemplateVariables(pod *corev1.Pod) EnvoyTemplateVari
169172
EnableAdminAccessForIpv6: m.mutatorConfig.enableAdminAccessIPv6,
170173
WaitUntilProxyReady: m.mutatorConfig.waitUntilProxyReady,
171174
UseFipsEndpoint: useFipsEndpoint,
175+
AwsAccessKeyId: m.mutatorConfig.awsAccessKeyId,
176+
AwsSecretAccessKey: m.mutatorConfig.awsSecretAccessKey,
177+
AwsSessionToken: m.mutatorConfig.awsSessionToken,
172178
}
173179
}
174180

‎pkg/inject/inject.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@ func (m *SidecarInjector) injectAppMeshPatches(ms *appmesh.Mesh, vn *appmesh.Vir
153153
postStartTimeout: m.config.PostStartTimeout,
154154
postStartInterval: m.config.PostStartInterval,
155155
useFipsEndpoint: m.config.FipsEndpoint,
156+
awsAccessKeyId: m.config.EnvoyAwsAccessKeyId,
157+
awsSecretAccessKey: m.config.EnvoyAwsSecretAccessKey,
158+
awsSessionToken: m.config.EnvoyAwsSessionToken,
156159
}, ms, vn),
157160
newXrayMutator(xrayMutatorConfig{
158161
awsRegion: m.awsRegion,
@@ -206,6 +209,9 @@ func (m *SidecarInjector) injectAppMeshPatches(ms *appmesh.Mesh, vn *appmesh.Vir
206209
useDualStackEndpoint: m.config.DualStackEndpoint,
207210
enableAdminAccessIPv6: m.config.EnvoyAdminAccessEnableIPv6,
208211
useFipsEndpoint: m.config.FipsEndpoint,
212+
awsAccessKeyId: m.config.EnvoyAwsAccessKeyId,
213+
awsSecretAccessKey: m.config.EnvoyAwsSecretAccessKey,
214+
awsSessionToken: m.config.EnvoyAwsSessionToken,
209215
}, ms, vg),
210216
newXrayMutator(xrayMutatorConfig{
211217
awsRegion: m.awsRegion,

‎pkg/inject/sidecar_builder.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ type EnvoyTemplateVariables struct {
5151
UseDualStackEndpoint string
5252
WaitUntilProxyReady bool
5353
UseFipsEndpoint string
54+
AwsAccessKeyId string
55+
AwsSecretAccessKey string
56+
AwsSessionToken string
5457
}
5558

5659
func updateEnvMapForEnvoy(vars EnvoyTemplateVariables, env map[string]string, vname string) error {
@@ -60,6 +63,18 @@ func updateEnvMapForEnvoy(vars EnvoyTemplateVariables, env map[string]string, vn
6063
env["APPMESH_VIRTUAL_NODE_NAME"] = vname
6164
env["AWS_REGION"] = vars.AWSRegion
6265

66+
// For usage outside traditional EC2 / Fargate IAM based profiles, this is needed to
67+
// propagate permissions to envoy. This is a rare use-case that's mostly just for testing.
68+
if len(vars.AwsAccessKeyId) > 0 {
69+
env["AWS_ACCESS_KEY_ID"] = vars.AwsAccessKeyId
70+
}
71+
if len(vars.AwsSecretAccessKey) > 0 {
72+
env["AWS_SECRET_ACCESS_KEY"] = vars.AwsSecretAccessKey
73+
}
74+
if len(vars.AwsSessionToken) > 0 {
75+
env["AWS_SESSION_TOKEN"] = vars.AwsSessionToken
76+
}
77+
6378
env["ENVOY_ADMIN_ACCESS_ENABLE_IPV6"] = strconv.FormatBool(vars.EnableAdminAccessForIpv6)
6479

6580
env["APPMESH_DUALSTACK_ENDPOINT"] = vars.UseDualStackEndpoint

0 commit comments

Comments
 (0)