waitUntilProxyReady feature (#598)

* Wait for envoy to initialize

* Move envoy container to front of list of pod app containers
* Update tests to reflect change in ordering

* Validate behavior with integration test

* Implement postStart hook

* Validate server_info LIVE does not guarantee envoy mesh

* Validate integration test by checking annotation

* Verison control integration test docker images

* Provision role and rolebinding

* Refactor provisioning functions to use kubernetes SDK

* More refactoring of test

* Use Job instead of Deployment

* Add feature flag

* Unit test feature flag

* Use existing vars instead of passing waitUntilProxyStarts flag
individually

* Add config to test.yaml

* Fix nit

* Fix unit test

* Set backofflimit to 1

* Add sidecar feature controller in CI integration tests

* Fix typo

* Add default for feature flag to false in Makefile

* Pass as command line arg not environment var

* Wait for controller upgrade

* Support agent poll envoyReadiness command

* Rename from wait for start to ready

* Fix unit test

* Update helm README

* Add integration test validating PostStart with Envoy v1.22

* Run without setting sidecar image tag

* Set sidecar image tag

* Validate all containers are ready

* Fix unit test

* Parse and check envoy version

* Add comment

* Fix unit test

* Add env vars to start agent server

* Regex version to 1.22.2-appmesh.1

* Enable sidecar test suite

* Properly pass config postStart values

* Fix unit tests

* Fix bash rematch condition

* Assign sidecar image repo and tag configs separately

* Clean up argument passing

* Default sidecar image tag

* Rename v1.22 sidecar test suite namespace

* Properly cleanup sidecar test resources

Co-authored-by: Kwangu Paul Yeo <[email protected]>

Author: paulyeo21

Makefile

@@ -8,6 +8,8 @@ VERSION ?= $(shell git describe --dirty --tags --always)
 IMAGE ?= $(REPO):$(VERSION)
 PREVIEW=false
 ENABLE_BACKEND_GROUPS?=false
+WAIT_PROXY_READY=false
+SIDECAR_IMAGE_TAG=v1.22.2.1-prod
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1"
@@ -56,9 +58,14 @@ deploy: check-env manifests
 helm-lint:
 	${MAKEFILE_PATH}/test/helm/helm-lint.sh
 
-
 helm-deploy: check-env manifests
-	helm upgrade -i appmesh-controller config/helm/appmesh-controller --namespace appmesh-system --set image.repository=$(REPO) --set image.tag=$(VERSION) --set preview=$(PREVIEW) --set enableBackendGroups=$(ENABLE_BACKEND_GROUPS)
+	helm upgrade -i appmesh-controller config/helm/appmesh-controller --namespace appmesh-system \
+		--set image.repository=$(REPO) \
+		--set image.tag=$(VERSION) \
+		--set preview=$(PREVIEW) \
+		--set enableBackendGroups=$(ENABLE_BACKEND_GROUPS) \
+		--set sidecar.waitUntilProxyReady=$(WAIT_PROXY_READY) \
+		--set sidecar.image.tag=$(SIDECAR_IMAGE_TAG)
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen

config/helm/appmesh-controller/README.md

@@ -390,8 +390,11 @@ Parameter | Description | Default
 `sidecar.resources.requests` | Envoy container resource requests | `requests: cpu 10m memory 32Mi`
 `sidecar.resources.limits` | Envoy container resource limits | `limits: cpu "" memory ""`
 `sidecar.lifecycleHooks.preStopDelay` | Envoy container PreStop Hook Delay Value | `20s`
+`sidecar.lifecycleHooks.postStartInterval` | Envoy container PostStart Hook Interval Value | `5s`
+`sidecar.lifecycleHooks.postStartTimeout` | Envoy container PostStart Hook Timeout Value | `180s`
 `sidecar.probes.readinessProbeInitialDelay` | Envoy container Readiness Probe Initial Delay | `1s`
 `sidecar.probes.readinessProbePeriod` | Envoy container Readiness Probe Period | `10s`
+`sidecar.waitUntilProxyReady` | Enable pod postStart hook to delay application startup until proxy is ready to accept traffic | `false`
 `init.image.repository` | Route manager image repository | `840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-proxy-route-manager`
 `init.image.tag` | Route manager image tag | `<VERSION>`
 `stats.tagsEnabled` |  If `true`, Envoy should include app-mesh tags | `false`

config/helm/appmesh-controller/templates/deployment.yaml

@@ -56,14 +56,17 @@ spec:
         args:
         - --enable-leader-election=true
         - --log-level={{ .Values.log.level }}
-        - --sidecar-image={{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}
+        - --sidecar-image-repository={{ .Values.sidecar.image.repository }}
+        - --sidecar-image-tag={{ .Values.sidecar.image.tag }}
         - --sidecar-cpu-requests={{ .Values.sidecar.resources.requests.cpu }}
         - --sidecar-memory-requests={{ .Values.sidecar.resources.requests.memory }}
         - --sidecar-cpu-limits={{ .Values.sidecar.resources.limits.cpu }}
         - --sidecar-memory-limits={{ .Values.sidecar.resources.limits.memory }}
         - --init-image={{ .Values.init.image.repository }}:{{ .Values.init.image.tag }}
         - --enable-stats-tags={{ .Values.stats.tagsEnabled }}
         - --prestop-delay={{ .Values.sidecar.lifecycleHooks.preStopDelay }}
+        - --poststart-timeout={{ .Values.sidecar.lifecycleHooks.postStartTimeout }}
+        - --poststart-interval={{ .Values.sidecar.lifecycleHooks.postStartInterval }}
         - --readiness-probe-initial-delay={{ .Values.sidecar.probes.readinessProbeInitialDelay }}
         - --readiness-probe-period={{ .Values.sidecar.probes.readinessProbePeriod }}
         - --envoy-admin-access-port={{ .Values.sidecar.envoyAdminAccessPort }}
@@ -114,6 +117,7 @@ spec:
         - --sidecar-log-level={{ .Values.sidecar.logLevel }}
         # this must be same as livenessProbe port which can be configured 
         - --health-probe-port={{ .Values.livenessProbe.httpGet.port }}
+        - --wait-until-proxy-ready={{ .Values.sidecar.waitUntilProxyReady }}
         {{- if .Values.env }}
         env:
           {{- range $key, $value := .Values.env }}

config/helm/appmesh-controller/test.yaml

@@ -35,10 +35,13 @@ sidecar:
   lifecycleHooks:
     # sidecar.lifecycleHooks: Envoy PreStop Hook Delay
     preStopDelay: 20
+    postStartTimeout: 180
+    postStartInterval: 5
   probes:
     # sidecar.probes: Envoy Readiness Probe
     readinessProbeInitialDelay: 1
     readinessProbePeriod: 10
+  waitUntilProxyReady: false
 init:
   image:
     repository: 840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-proxy-route-manager

config/helm/appmesh-controller/values.yaml

@@ -36,10 +36,13 @@ sidecar:
   lifecycleHooks:
     # sidecar.lifecycleHooks: Envoy PreStop Hook Delay
     preStopDelay: 20
+    postStartInterval: 5
+    postStartTimeout: 180
   probes:
     # sidecar.probes: Envoy Readiness Probe
     readinessProbeInitialDelay: 1
     readinessProbePeriod: 10
+  waitUntilProxyReady: false
 init:
   image:
     repository: 840364872350.dkr.ecr.us-west-2.amazonaws.com/aws-appmesh-proxy-route-manager

pkg/inject/config.go

@@ -13,20 +13,24 @@ const (
 	flagSdsUdsPath                  = "sds-uds-path"
 	flagEnableBackendGroups         = "enable-backend-groups"
 
-	flagSidecarImage               = "sidecar-image"
+	flagSidecarImageRepository     = "sidecar-image-repository"
+	flagSidecarImageTag            = "sidecar-image-tag"
 	flagSidecarCpuRequests         = "sidecar-cpu-requests"
 	flagSidecarMemoryRequests      = "sidecar-memory-requests"
 	flagSidecarCpuLimits           = "sidecar-cpu-limits"
 	flagSidecarMemoryLimits        = "sidecar-memory-limits"
 	flagPreview                    = "preview"
 	flagLogLevel                   = "sidecar-log-level"
 	flagPreStopDelay               = "prestop-delay"
+	flagPostStartTimeout           = "poststart-timeout"
+	flagPostStartInterval          = "poststart-interval"
 	flagReadinessProbeInitialDelay = "readiness-probe-initial-delay"
 	flagReadinessProbePeriod       = "readiness-probe-period"
 	flagEnvoyAdminAccessPort       = "envoy-admin-access-port"
 	flagEnvoyAdminAccessLogFile    = "envoy-admin-access-log-file"
 	flagEnvoyAdminAccessEnableIpv6 = "envoy-admin-access-enable-ipv6"
 	flagDualStackEndpoint          = "dual-stack-endpoint"
+	flagWaitUntilProxyReady        = "wait-until-proxy-ready"
 
 	flagInitImage  = "init-image"
 	flagIgnoredIPs = "ignored-ips"
@@ -66,20 +70,24 @@ type Config struct {
 	EnableBackendGroups bool
 
 	// Sidecar settings
-	SidecarImage               string
+	SidecarImageRepository     string
+	SidecarImageTag            string
 	SidecarCpuRequests         string
 	SidecarMemoryRequests      string
 	SidecarCpuLimits           string
 	SidecarMemoryLimits        string
 	Preview                    bool
 	LogLevel                   string
 	PreStopDelay               string
+	PostStartTimeout           int32
+	PostStartInterval          int32
 	ReadinessProbeInitialDelay int32
 	ReadinessProbePeriod       int32
 	EnvoyAdminAcessPort        int32
 	EnvoyAdminAccessLogFile    string
 	DualStackEndpoint          bool
 	EnvoyAdminAccessEnableIPv6 bool
+	WaitUntilProxyReady        bool
 
 	// Init container settings
 	InitImage  string
@@ -127,8 +135,9 @@ func (cfg *Config) BindFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&cfg.SdsUdsPath, flagSdsUdsPath, "/run/spire/sockets/agent.sock",
 		"Unix Domain Socket path for SDS provider")
 	fs.BoolVar(&cfg.EnableBackendGroups, flagEnableBackendGroups, false, "If enabled, experimental Backend Groups feature will be enabled.")
-	fs.StringVar(&cfg.SidecarImage, flagSidecarImage, "public.ecr.aws/appmesh/aws-appmesh-envoy:v1.22.2.1-prod",
-		"Envoy sidecar container image.")
+	fs.StringVar(&cfg.SidecarImageRepository, flagSidecarImageRepository, "public.ecr.aws/appmesh/aws-appmesh-envoy",
+		"Envoy sidecar container image repository.")
+	fs.StringVar(&cfg.SidecarImageTag, flagSidecarImageTag, "v1.22.2.1-prod", "Envoy sidecar container image tag.")
 	fs.StringVar(&cfg.SidecarCpuRequests, flagSidecarCpuRequests, "10m",
 		"Sidecar CPU resources requests.")
 	fs.StringVar(&cfg.SidecarMemoryRequests, flagSidecarMemoryRequests, "32Mi",
@@ -147,6 +156,10 @@ func (cfg *Config) BindFlags(fs *pflag.FlagSet) {
 		"AWS App Mesh envoy access log path")
 	fs.StringVar(&cfg.PreStopDelay, flagPreStopDelay, "20",
 		"AWS App Mesh envoy preStop hook sleep duration")
+	fs.Int32Var(&cfg.PostStartTimeout, flagPostStartTimeout, 180,
+		"AWS App Mesh envoy postStart hook timeout duration")
+	fs.Int32Var(&cfg.PostStartInterval, flagPostStartInterval, 5,
+		"AWS App Mesh envoy postStart hook interval duration")
 	fs.Int32Var(&cfg.ReadinessProbeInitialDelay, flagReadinessProbeInitialDelay, 1,
 		"Number of seconds after Envoy has started before readiness probes are initiated")
 	fs.Int32Var(&cfg.ReadinessProbePeriod, flagReadinessProbePeriod, 10,
@@ -192,6 +205,8 @@ func (cfg *Config) BindFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&cfg.DualStackEndpoint, flagDualStackEndpoint, false, "Use DualStack Endpoint")
 	fs.BoolVar(&cfg.DualStackEndpoint, flagEnvoyAdminAccessEnableIpv6, false, "Enable Admin access when using IPv6")
 	fs.StringVar(&cfg.ClusterName, flagClusterName, "", "ClusterName in context")
+	fs.BoolVar(&cfg.WaitUntilProxyReady, flagWaitUntilProxyReady, false,
+		"Enable pod postStart hook to delay application startup until proxy is ready to accept traffic")
 }
 
 func (cfg *Config) BindEnv() error {

pkg/inject/envoy.go

@@ -23,9 +23,12 @@ type envoyMutatorConfig struct {
 	adminAccessPort            int32
 	adminAccessLogFile         string
 	preStopDelay               string
+	postStartTimeout           int32
+	postStartInterval          int32
 	readinessProbeInitialDelay int32
 	readinessProbePeriod       int32
-	sidecarImage               string
+	sidecarImageRepository     string
+	sidecarImageTag            string
 	sidecarCPURequests         string
 	sidecarMemoryRequests      string
 	sidecarCPULimits           string
@@ -44,6 +47,7 @@ type envoyMutatorConfig struct {
 	statsDPort                 int32
 	statsDAddress              string
 	statsDSocketPath           string
+	waitUntilProxyReady        bool
 	controllerVersion          string
 	k8sVersion                 string
 	useDualStackEndpoint       bool
@@ -107,7 +111,14 @@ func (m *envoyMutator) mutate(pod *corev1.Pod) error {
 	if m.mutatorConfig.enableSDS && !isSDSDisabled(pod) {
 		mutateSDSMounts(pod, &container, m.mutatorConfig.sdsUdsPath)
 	}
-	pod.Spec.Containers = append(pod.Spec.Containers, container)
+
+	// waitUntilProxyReady requires starting sidecar container first
+	if m.mutatorConfig.waitUntilProxyReady {
+		pod.Spec.Containers = append([]corev1.Container{container}, pod.Spec.Containers...)
+	} else {
+		pod.Spec.Containers = append(pod.Spec.Containers, container)
+	}
+
 	return nil
 }
 
@@ -132,7 +143,10 @@ func (m *envoyMutator) buildTemplateVariables(pod *corev1.Pod) EnvoyTemplateVari
 		AdminAccessPort:          m.mutatorConfig.adminAccessPort,
 		AdminAccessLogFile:       m.mutatorConfig.adminAccessLogFile,
 		PreStopDelay:             m.mutatorConfig.preStopDelay,
-		SidecarImage:             m.mutatorConfig.sidecarImage,
+		PostStartTimeout:         m.mutatorConfig.postStartTimeout,
+		PostStartInterval:        m.mutatorConfig.postStartInterval,
+		SidecarImageRepository:   m.mutatorConfig.sidecarImageRepository,
+		SidecarImageTag:          m.mutatorConfig.sidecarImageTag,
 		EnableXrayTracing:        m.mutatorConfig.enableXrayTracing,
 		XrayDaemonPort:           m.mutatorConfig.xrayDaemonPort,
 		XraySamplingRate:         m.mutatorConfig.xraySamplingRate,
@@ -151,6 +165,7 @@ func (m *envoyMutator) buildTemplateVariables(pod *corev1.Pod) EnvoyTemplateVari
 		K8sVersion:               m.mutatorConfig.k8sVersion,
 		UseDualStackEndpoint:     useDualStackEndpoint,
 		EnableAdminAccessForIpv6: m.mutatorConfig.enableAdminAccessIPv6,
+		WaitUntilProxyReady:      m.mutatorConfig.waitUntilProxyReady,
 	}
 }