fix(controller): tolerate delayed webhook certificate startup

Keep process liveness independent of certificate readiness and allow a bounded startup window in the Helm chart. Preserve certificate and webhook gating for readiness.

Add simple unit coverage for disabled webhooks, pending certificates, webhook startup, and readiness. Do not add E2E coverage or readiness diagnostics.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 0c077ebb-9cd5-4c85-a49d-0178a7f67a57

Author: joshualong_microsoft

src/subsystems/cache-controller/charts/dacs/templates/manager/manager.yaml

@@ -99,6 +99,12 @@ spec:
                         port: 8081
                     initialDelaySeconds: 5
                     periodSeconds: 10
+                  startupProbe:
+                    httpGet:
+                        path: /readyz
+                        port: 8081
+                    failureThreshold: 60
+                    periodSeconds: 5
                   resources:
                     {{- if .Values.manager.resources }}
                     {{- toYaml .Values.manager.resources | nindent 20 }}

src/subsystems/cache-controller/cmd/main.go

@@ -402,30 +402,48 @@ func main() {
 
 	// +kubebuilder:scaffold:builder
 
-	// Setup health checks with cert readiness awareness
+	if err := setupHealthChecks(mgr, enableWebhooks, certSetupFinished, func(req *http.Request) error {
+		return mgr.GetWebhookServer().StartedChecker()(req)
+	}); err != nil {
+		logAndExit(err, "unable to set up health checks")
+	}
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctx); err != nil {
+		logAndExit(err, "problem running manager")
+	}
+}
+
+type healthCheckRegistrar interface {
+	AddHealthzCheck(string, healthz.Checker) error
+	AddReadyzCheck(string, healthz.Checker) error
+}
+
+func setupHealthChecks(
+	mgr healthCheckRegistrar,
+	enableWebhooks bool,
+	certSetupFinished <-chan struct{},
+	webhookStarted healthz.Checker,
+) error {
 	checker := healthz.Ping
 	if enableWebhooks {
 		checker = func(req *http.Request) error {
 			select {
 			case <-certSetupFinished:
-				return mgr.GetWebhookServer().StartedChecker()(req)
+				return webhookStarted(req)
 			default:
 				return fmt.Errorf("certs are not ready yet")
 			}
 		}
 	}
 
-	if err := mgr.AddHealthzCheck("healthz", checker); err != nil {
-		logAndExit(err, "unable to set up health check")
+	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
+		return fmt.Errorf("registering liveness check: %w", err)
 	}
 	if err := mgr.AddReadyzCheck("readyz", checker); err != nil {
-		logAndExit(err, "unable to set up ready check")
-	}
-
-	setupLog.Info("starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		logAndExit(err, "problem running manager")
+		return fmt.Errorf("registering readiness check: %w", err)
 	}
+	return nil
 }
 
 // logAndExit logs the error and exits the program with a non-zero status code.

src/subsystems/cache-controller/cmd/main_test.go

@@ -15,11 +15,15 @@
 package main
 
 import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 
 	"go.uber.org/zap/zapcore"
 	"gotest.tools/v3/assert"
@@ -101,3 +105,69 @@ func TestRemoveUnusedNodeFields_NonNodeInput(t *testing.T) {
 	assert.NilError(t, err)
 	assert.Equal(t, any(cm), out, "expected pass-through for non-Node input")
 }
+
+type healthCheckRecorder struct {
+	checks map[string]healthz.Checker
+}
+
+func (r *healthCheckRecorder) AddHealthzCheck(name string, checker healthz.Checker) error {
+	r.checks[name] = checker
+	return nil
+}
+
+func (r *healthCheckRecorder) AddReadyzCheck(name string, checker healthz.Checker) error {
+	r.checks[name] = checker
+	return nil
+}
+
+func TestSetupHealthChecks(t *testing.T) {
+	tests := []struct {
+		name           string
+		enableWebhooks bool
+		certsReady     bool
+		webhookErr     error
+		wantErr        string
+	}{
+		{name: "webhooks disabled"},
+		{name: "certificates pending", enableWebhooks: true, wantErr: "certs are not ready yet"},
+		{
+			name:           "webhook starting",
+			enableWebhooks: true,
+			certsReady:     true,
+			webhookErr:     errors.New("webhook server has not started"),
+			wantErr:        "webhook server has not started",
+		},
+		{name: "webhook ready", enableWebhooks: true, certsReady: true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			certsReady := make(chan struct{})
+			if tt.certsReady {
+				close(certsReady)
+			}
+			recorder := &healthCheckRecorder{checks: make(map[string]healthz.Checker)}
+			webhookCalls := 0
+			err := setupHealthChecks(recorder, tt.enableWebhooks, certsReady, func(*http.Request) error {
+				webhookCalls++
+				return tt.webhookErr
+			})
+			assert.NilError(t, err)
+			assert.Assert(t, recorder.checks["healthz"] != nil, "liveness check must be registered")
+			assert.Assert(t, recorder.checks["readyz"] != nil, "readiness check must be registered")
+			req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/healthz", nil)
+			assert.NilError(t, recorder.checks["healthz"](req), "liveness must not depend on certificate or webhook readiness")
+			req = httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/readyz", nil)
+			err = recorder.checks["readyz"](req)
+			if tt.wantErr == "" {
+				assert.NilError(t, err)
+			} else {
+				assert.Error(t, err, tt.wantErr)
+			}
+			wantWebhookCalls := 0
+			if tt.enableWebhooks && tt.certsReady {
+				wantWebhookCalls = 1
+			}
+			assert.Equal(t, webhookCalls, wantWebhookCalls, "only check webhook startup after certificates are ready")
+		})
+	}
+}

src/subsystems/cache-controller/docs/cert-controller.md

@@ -34,17 +34,40 @@ This allows the Kubernetes API server to trust the webhook's TLS certificate.
 - **Path:** `/tmp/k8s-webhook-server/serving-certs`
 - **Purpose:** The webhook server reads certificates from this directory
 
-The rotator writes certificates here so the controller-runtime webhook server can use them.
+Kubelet projects the certificate Secret into this directory so the
+controller-runtime webhook server can use it. Secret updates are asynchronous;
+generating certificates does not make the mounted files available immediately.
 
 ## How It Works
 
 1. Controller starts -> cert rotator initializes
 2. Rotator generates self-signed CA + server certificate
-3. Certificates stored in Secret and written to filesystem
+3. Certificates stored in Secret and projected into the pod by kubelet
 4. CA bundle injected into webhook configurations
 5. Webhooks registered (only after certs are ready)
 6. Rotator monitors expiry and auto-renews
 
+## Health Probes
+
+`/healthz` checks process liveness independently of certificate initialization.
+`/readyz` succeeds only after certificate setup completes and the webhook server
+starts. With webhooks disabled, readiness does not depend on certificates.
+
+The Helm chart's startup probe checks `/readyz` every five seconds and allows
+60 failures, giving initialization approximately five minutes before Kubernetes
+restarts the container. Liveness and readiness probes begin after startup
+succeeds. Keep `EnableReadinessCheck` enabled in the cert rotator so webhook
+configuration reconciliation waits for mounted certificates.
+
+Unit tests cover the registered liveness and readiness checks, certificate
+initialization, and webhook startup. Run them from
+`src/subsystems/cache-controller`:
+
+```bash
+CACHE_CONTROLLER_CONFIG_PATH="$PWD/internal/config/cache-controller-config.yaml" \
+  go test ./cmd
+```
+
 ## RBAC Requirements
 
 | File | Purpose |
@@ -58,4 +81,3 @@ Check certificate secret:
 ```
 kubectl get secret cache-controller-webhook-server-cert -n cache-controller-system
 ```
-