feat: make reboot — diagnose + auto-heal the lab after a host reboot

Makefile

@@ -54,7 +54,7 @@ SECRET_KEY  ?= s3cr3t
 
 .DEFAULT_GOAL := help
 
-.PHONY: help deploy destroy nuke bootstrap preflight cluster-up floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean
+.PHONY: help deploy destroy nuke bootstrap preflight cluster-up reboot floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean
 
 help: ## Show this help
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN{FS=":.*?## "}{printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}'
@@ -130,6 +130,9 @@ floci-down: ## Stop the floci AWS emulator
 preflight: ## Fail fast if Docker can't push to the floci ECR registry over HTTP
 	@./scripts/preflight-docker-registry.sh
 
+reboot: ## Diagnose + auto-heal the lab after a host reboot (flannel, DNS, image, ESO)
+	@./scripts/recover.sh
+
 bootstrap: preflight floci-up ## Start floci + apply secrets->infra->secrets(KMS) in the required order
 	@echo "==> floci up; applying secrets (pass 1, no KMS)"
 	$(RETRY) 'cd $(TF_SECRETS) && $(TF) init -input=false && $(TF) apply $(APPROVE) -var="db_password=$(DB_PASSWORD)" -var="api_key=$(API_KEY)" -var="secret_key=$(SECRET_KEY)"'

README.md

@@ -488,6 +488,40 @@ helm diff upgrade webapp charts/webapp/ -n webapp
 
 ---
 
+## After a reboot: `make reboot`
+
+A host reboot restarts the k3d node container, which breaks several things a
+plain `kubectl get pods` won't explain — and they compound:
+
+| Symptom | Cause |
+| --- | --- |
+| Pods stuck `ContainerCreating`, `failed to setup network ... /run/flannel/subnet.env` | flannel's subnet file lives on tmpfs (`/run`), wiped on node restart |
+| ESO `SecretSyncedError`, `lookup host.k3d.internal ... i/o timeout` | k3s strips `host.k3d.internal` from CoreDNS on restart |
+| webapp `ImagePullBackOff`, `dial tcp 127.0.0.1:5100: connection refused` | the image dropped out of the node's containerd; the node can't pull the floci ECR ref |
+| ESO/webapp errors, floci unreachable | floci container down / lost un-flushed data |
+
+`make reboot` diagnoses and auto-heals all of them (idempotent, safe to run
+anytime — it never touches Terraform state, floci secrets, or the infra/secrets
+roots):
+
+```bash
+make reboot
+```
+
+It checks and fixes, in order: floci up → cluster started → **flannel** (bounces
+the node if `subnet.env` is missing) → **host.k3d.internal** (durable
+coredns-custom mapping) → **webapp image on the node** (see below) → ESO
+re-sync → log shipper → webapp rollout, then prints status and the app HTTP code.
+
+> **Image loading on modern Docker.** If `docker info` shows the
+> `containerd-snapshotter` image store (default on recent Docker/Ubuntu),
+> `k3d image import` silently no-ops (its `docker save` output is an OCI layout
+> the importer rejects: *"content digest not found"*). `make deploy` and
+> `make reboot` both use `scripts/ensure-webapp-image.sh`, which verifies the
+> image actually landed and falls back to pulling the identical upstream image
+> into the node's containerd and retagging it. So the image step is reliable
+> regardless of Docker's storage driver.
+
 ## Key Commands
 
 ```bash

scripts/ensure-webapp-image.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Ensure the webapp image is present in the k3d node's containerd, so the pod
+# (imagePullPolicy: IfNotPresent) never tries to pull it. The node CANNOT pull
+# the floci ECR ref at runtime — inside the node that hostname resolves to the
+# node's own loopback — so the image must be loaded onto the node out-of-band.
+#
+# Two load paths, tried in order, because `k3d image import` silently no-ops on
+# Docker's containerd-snapshotter image store (the default on recent Docker /
+# Ubuntu): `docker save` there emits an OCI layout that the importer rejects with
+# "content digest ... not found", yet reports success. So we verify after import
+# and fall back to pulling the identical upstream image straight into the node's
+# containerd (the node has internet even when the pod CNI is down) and retagging.
+#
+# Idempotent. Usage: ensure-webapp-image.sh [cluster-name]
+set -u
+
+CLUSTER="${1:-webapp-test}"
+NODE="k3d-${CLUSTER}-server-0"
+REF="000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27"
+# The ECR image is a straight retag of this upstream image (see terraform/infra/
+# ecr.tf — pull + tag + push, no build), so the content is identical.
+SRC="docker.io/nginxinc/nginx-unprivileged:1.27"
+
+present() { docker exec "$NODE" crictl images 2>/dev/null | grep -q 'webapp/nginx'; }
+
+if present; then
+	echo "ensure-image: $REF already on $NODE"
+	exit 0
+fi
+
+# Path 1: k3d image import (works on the classic overlay2/docker image store).
+echo "ensure-image: image missing — trying 'k3d image import'"
+k3d image import "$REF" -c "$CLUSTER" >/dev/null 2>&1 || true
+if present; then
+	echo "ensure-image: loaded via k3d image import"
+	exit 0
+fi
+
+# Path 2: containerd-snapshotter store breaks k3d/ctr import — pull upstream into
+# the node's containerd and retag as the ECR ref the Deployment references.
+echo "ensure-image: k3d import didn't land (containerd-snapshotter store?) — pulling upstream + retag"
+docker exec "$NODE" crictl pull "$SRC"
+docker exec "$NODE" ctr -n k8s.io images tag "$SRC" "$REF" 2>/dev/null || true
+if present; then
+	echo "ensure-image: loaded via pull+retag"
+	exit 0
+fi
+
+echo "ensure-image: FAILED to load $REF onto $NODE" >&2
+exit 1

scripts/recover.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# make reboot — diagnose and auto-heal the lab after a host reboot.
+#
+# A reboot restarts the k3d node container, which reliably breaks several things
+# that a plain `kubectl get pods` won't explain:
+#   1. floci (the AWS emulator) may be down / lost un-flushed data
+#   2. the k3d cluster is stopped
+#   3. flannel's /run/flannel/subnet.env is on tmpfs -> wiped -> pod sandboxes
+#      fail ("failed to setup network ... open /run/flannel/subnet.env")
+#   4. host.k3d.internal is stripped from CoreDNS -> ESO can't reach floci
+#      ("dial tcp: lookup host.k3d.internal ... i/o timeout")
+#   5. the webapp image can drop out of the node's containerd -> ImagePullBackOff
+#
+# This script checks each, reports what it finds, and fixes it. Idempotent and
+# safe to run anytime (not just after a reboot). It does NOT touch Terraform
+# state, floci secrets, or the infra/secrets roots.
+set -u
+
+CLUSTER="webapp-test"
+CTX="k3d-${CLUSTER}"
+NODE="k3d-${CLUSTER}-server-0"
+KUBECTL="kubectl --context ${CTX}"
+DIR="$(cd "$(dirname "$0")" && pwd)"
+ROOT="$(cd "$DIR/.." && pwd)"
+
+step() { printf '\n=== %s ===\n' "$1"; }
+ok()   { printf '  [ok]  %s\n' "$1"; }
+fix()  { printf '  [fix] %s\n' "$1"; }
+warn() { printf '  [!]   %s\n' "$1"; }
+
+# 1. floci ------------------------------------------------------------------
+step "floci AWS emulator (:4566)"
+if curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1; then
+	ok "floci reachable"
+else
+	fix "floci down — starting via docker compose"
+	(cd "$ROOT" && docker compose -p floci up -d >/dev/null 2>&1) || true
+	for i in $(seq 1 30); do
+		curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && break
+		sleep 2
+	done
+	curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && ok "floci back up" || warn "floci still unreachable — check 'docker compose -p floci ps'"
+fi
+
+# 2. cluster running --------------------------------------------------------
+step "k3d cluster"
+if ! k3d cluster list "$CLUSTER" >/dev/null 2>&1; then
+	warn "cluster '$CLUSTER' does not exist — run 'make deploy' first"
+	exit 1
+fi
+if docker inspect -f '{{.State.Running}}' "$NODE" 2>/dev/null | grep -q true; then
+	ok "node container running"
+else
+	fix "node stopped — starting cluster"
+	k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true
+fi
+
+# 3. node Ready + flannel CNI ----------------------------------------------
+step "node readiness + flannel CNI"
+$KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true
+if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then
+	ok "flannel subnet.env present"
+else
+	fix "flannel subnet.env missing — bouncing node to reinitialize CNI"
+	k3d cluster stop "$CLUSTER" >/dev/null 2>&1 && k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true
+	$KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true
+	if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then
+		ok "flannel recovered"
+	else
+		warn "flannel still missing — inspect 'docker logs $NODE'"
+	fi
+fi
+
+# 4. host.k3d.internal (ESO -> floci) --------------------------------------
+step "host.k3d.internal resolution (ESO -> floci)"
+fix "re-asserting host.k3d.internal (durable coredns-custom mapping)"
+"$DIR/coredns-hostfix.sh" "$CLUSTER" >/dev/null 2>&1 && ok "coredns host-fix applied" || warn "coredns-hostfix.sh failed"
+
+# 5. webapp image on the node ----------------------------------------------
+step "webapp image on node"
+"$DIR/ensure-webapp-image.sh" "$CLUSTER" || warn "could not load webapp image — see output above"
+
+# 6. External Secrets re-sync ----------------------------------------------
+step "External Secrets (force re-sync from floci)"
+$KUBECTL -n external-secrets rollout restart deploy/external-secrets >/dev/null 2>&1 || true
+$KUBECTL -n external-secrets rollout status deploy/external-secrets --timeout=90s >/dev/null 2>&1 || true
+ok "ESO controller restarted"
+
+# 7. log shipper ------------------------------------------------------------
+step "CloudWatch log shipper"
+if $KUBECTL apply -f "$ROOT/logging/cloudwatch-log-shipper.yaml" >/dev/null 2>&1; then
+	ok "log-shipper applied"
+else
+	warn "log-shipper apply failed"
+fi
+
+# 8. nudge the webapp workload ---------------------------------------------
+step "webapp workload"
+$KUBECTL -n webapp delete pod -l app=webapp >/dev/null 2>&1 || true
+$KUBECTL -n webapp rollout status deploy/webapp --timeout=120s >/dev/null 2>&1 || true
+
+# 9. final status -----------------------------------------------------------
+step "status"
+$KUBECTL -n webapp get pods 2>/dev/null || true
+echo
+$KUBECTL -n webapp get externalsecret 2>/dev/null || true
+code="$(curl -s -o /dev/null -w '%{http_code}' http://localhost:30080 2>/dev/null)"
+echo
+if [ "$code" = "200" ]; then
+	ok "app responding: HTTP $code at http://localhost:30080"
+else
+	warn "app HTTP $code (expected 200). If ExternalSecret shows SecretSyncedError,"
+	warn "floci may have lost its secrets on reboot — reseed with 'make bootstrap'."
+fi

terraform/cluster/webapp.tf

@@ -12,7 +12,11 @@
 resource "null_resource" "ecr_image_import" {
   depends_on = [time_sleep.cluster_ready]
 
+  # Load the image onto the node robustly. `k3d image import` silently no-ops on
+  # Docker's containerd-snapshotter store (default on recent Docker/Ubuntu), so
+  # this helper verifies the image actually landed and falls back to a
+  # pull+retag if not. See scripts/ensure-webapp-image.sh.
   provisioner "local-exec" {
-    command = "k3d image import 000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27 -c webapp-test"
+    command = "${path.module}/../../scripts/ensure-webapp-image.sh webapp-test"
   }
 }