From 3543a8c157ecfd1cd2d4881195e1b11df28419ae Mon Sep 17 00:00:00 2001 From: leehart32 Date: Sat, 11 Jul 2026 11:54:57 -0600 Subject: [PATCH] =?UTF-8?q?feat:=20make=20reboot=20=E2=80=94=20diagnose=20?= =?UTF-8?q?+=20auto-heal=20the=20lab=20after=20a=20host=20reboot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host reboot restarts the k3d node container and breaks several things that a plain `kubectl get pods` won't explain, and they compound: - flannel /run/flannel/subnet.env wiped (tmpfs) -> pod sandbox failures - host.k3d.internal stripped from CoreDNS -> ESO can't reach floci - webapp image drops out of node containerd -> ImagePullBackOff (node can't pull the floci ECR ref: inside the node it resolves to the node's own loopback) - floci down / lost un-flushed data New: - scripts/recover.sh (make reboot): checks and fixes each of the above in order, idempotent, never touches Terraform state / floci secrets / infra roots. - scripts/ensure-webapp-image.sh: snapshotter-safe image loader. `k3d image import` silently no-ops on Docker's containerd-snapshotter store (default on recent Docker/Ubuntu) — docker save emits an OCI layout the importer rejects ("content digest not found") but reports success. This verifies the image landed and falls back to pulling the identical upstream image into the node's containerd + retagging. Used by both `make reboot` and the deploy. - terraform/cluster/webapp.tf: image import now calls ensure-webapp-image.sh, so a fresh `make deploy` works regardless of Docker's storage driver. - README: "After a reboot" section + the image-loading note. Validated end-to-end on a live cluster: healthy checks pass, host-fix reapplied, ESO re-synced (SecretSynced), webapp 1/1, app HTTP 200. Security gate PASS. Co-Authored-By: Claude Opus 4.8 --- Makefile | 5 +- README.md | 34 ++++++++++ scripts/ensure-webapp-image.sh | 50 +++++++++++++++ scripts/recover.sh | 114 +++++++++++++++++++++++++++++++++ terraform/cluster/webapp.tf | 6 +- 5 files changed, 207 insertions(+), 2 deletions(-) create mode 100755 scripts/ensure-webapp-image.sh create mode 100755 scripts/recover.sh diff --git a/Makefile b/Makefile index 28cfb6b..0e217d7 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ SECRET_KEY ?= s3cr3t .DEFAULT_GOAL := help -.PHONY: help deploy destroy nuke bootstrap preflight cluster-up floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean +.PHONY: help deploy destroy nuke bootstrap preflight cluster-up reboot floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' @@ -130,6 +130,9 @@ floci-down: ## Stop the floci AWS emulator preflight: ## Fail fast if Docker can't push to the floci ECR registry over HTTP @./scripts/preflight-docker-registry.sh +reboot: ## Diagnose + auto-heal the lab after a host reboot (flannel, DNS, image, ESO) + @./scripts/recover.sh + bootstrap: preflight floci-up ## Start floci + apply secrets->infra->secrets(KMS) in the required order @echo "==> floci up; applying secrets (pass 1, no KMS)" $(RETRY) 'cd $(TF_SECRETS) && $(TF) init -input=false && $(TF) apply $(APPROVE) -var="db_password=$(DB_PASSWORD)" -var="api_key=$(API_KEY)" -var="secret_key=$(SECRET_KEY)"' diff --git a/README.md b/README.md index 6e139f4..42c931e 100644 --- a/README.md +++ b/README.md @@ -488,6 +488,40 @@ helm diff upgrade webapp charts/webapp/ -n webapp --- +## After a reboot: `make reboot` + +A host reboot restarts the k3d node container, which breaks several things a +plain `kubectl get pods` won't explain — and they compound: + +| Symptom | Cause | +| --- | --- | +| Pods stuck `ContainerCreating`, `failed to setup network ... /run/flannel/subnet.env` | flannel's subnet file lives on tmpfs (`/run`), wiped on node restart | +| ESO `SecretSyncedError`, `lookup host.k3d.internal ... i/o timeout` | k3s strips `host.k3d.internal` from CoreDNS on restart | +| webapp `ImagePullBackOff`, `dial tcp 127.0.0.1:5100: connection refused` | the image dropped out of the node's containerd; the node can't pull the floci ECR ref | +| ESO/webapp errors, floci unreachable | floci container down / lost un-flushed data | + +`make reboot` diagnoses and auto-heals all of them (idempotent, safe to run +anytime — it never touches Terraform state, floci secrets, or the infra/secrets +roots): + +```bash +make reboot +``` + +It checks and fixes, in order: floci up → cluster started → **flannel** (bounces +the node if `subnet.env` is missing) → **host.k3d.internal** (durable +coredns-custom mapping) → **webapp image on the node** (see below) → ESO +re-sync → log shipper → webapp rollout, then prints status and the app HTTP code. + +> **Image loading on modern Docker.** If `docker info` shows the +> `containerd-snapshotter` image store (default on recent Docker/Ubuntu), +> `k3d image import` silently no-ops (its `docker save` output is an OCI layout +> the importer rejects: *"content digest not found"*). `make deploy` and +> `make reboot` both use `scripts/ensure-webapp-image.sh`, which verifies the +> image actually landed and falls back to pulling the identical upstream image +> into the node's containerd and retagging it. So the image step is reliable +> regardless of Docker's storage driver. + ## Key Commands ```bash diff --git a/scripts/ensure-webapp-image.sh b/scripts/ensure-webapp-image.sh new file mode 100755 index 0000000..ebf94ae --- /dev/null +++ b/scripts/ensure-webapp-image.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Ensure the webapp image is present in the k3d node's containerd, so the pod +# (imagePullPolicy: IfNotPresent) never tries to pull it. The node CANNOT pull +# the floci ECR ref at runtime — inside the node that hostname resolves to the +# node's own loopback — so the image must be loaded onto the node out-of-band. +# +# Two load paths, tried in order, because `k3d image import` silently no-ops on +# Docker's containerd-snapshotter image store (the default on recent Docker / +# Ubuntu): `docker save` there emits an OCI layout that the importer rejects with +# "content digest ... not found", yet reports success. So we verify after import +# and fall back to pulling the identical upstream image straight into the node's +# containerd (the node has internet even when the pod CNI is down) and retagging. +# +# Idempotent. Usage: ensure-webapp-image.sh [cluster-name] +set -u + +CLUSTER="${1:-webapp-test}" +NODE="k3d-${CLUSTER}-server-0" +REF="000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27" +# The ECR image is a straight retag of this upstream image (see terraform/infra/ +# ecr.tf — pull + tag + push, no build), so the content is identical. +SRC="docker.io/nginxinc/nginx-unprivileged:1.27" + +present() { docker exec "$NODE" crictl images 2>/dev/null | grep -q 'webapp/nginx'; } + +if present; then + echo "ensure-image: $REF already on $NODE" + exit 0 +fi + +# Path 1: k3d image import (works on the classic overlay2/docker image store). +echo "ensure-image: image missing — trying 'k3d image import'" +k3d image import "$REF" -c "$CLUSTER" >/dev/null 2>&1 || true +if present; then + echo "ensure-image: loaded via k3d image import" + exit 0 +fi + +# Path 2: containerd-snapshotter store breaks k3d/ctr import — pull upstream into +# the node's containerd and retag as the ECR ref the Deployment references. +echo "ensure-image: k3d import didn't land (containerd-snapshotter store?) — pulling upstream + retag" +docker exec "$NODE" crictl pull "$SRC" +docker exec "$NODE" ctr -n k8s.io images tag "$SRC" "$REF" 2>/dev/null || true +if present; then + echo "ensure-image: loaded via pull+retag" + exit 0 +fi + +echo "ensure-image: FAILED to load $REF onto $NODE" >&2 +exit 1 diff --git a/scripts/recover.sh b/scripts/recover.sh new file mode 100755 index 0000000..1d67441 --- /dev/null +++ b/scripts/recover.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# make reboot — diagnose and auto-heal the lab after a host reboot. +# +# A reboot restarts the k3d node container, which reliably breaks several things +# that a plain `kubectl get pods` won't explain: +# 1. floci (the AWS emulator) may be down / lost un-flushed data +# 2. the k3d cluster is stopped +# 3. flannel's /run/flannel/subnet.env is on tmpfs -> wiped -> pod sandboxes +# fail ("failed to setup network ... open /run/flannel/subnet.env") +# 4. host.k3d.internal is stripped from CoreDNS -> ESO can't reach floci +# ("dial tcp: lookup host.k3d.internal ... i/o timeout") +# 5. the webapp image can drop out of the node's containerd -> ImagePullBackOff +# +# This script checks each, reports what it finds, and fixes it. Idempotent and +# safe to run anytime (not just after a reboot). It does NOT touch Terraform +# state, floci secrets, or the infra/secrets roots. +set -u + +CLUSTER="webapp-test" +CTX="k3d-${CLUSTER}" +NODE="k3d-${CLUSTER}-server-0" +KUBECTL="kubectl --context ${CTX}" +DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$DIR/.." && pwd)" + +step() { printf '\n=== %s ===\n' "$1"; } +ok() { printf ' [ok] %s\n' "$1"; } +fix() { printf ' [fix] %s\n' "$1"; } +warn() { printf ' [!] %s\n' "$1"; } + +# 1. floci ------------------------------------------------------------------ +step "floci AWS emulator (:4566)" +if curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1; then + ok "floci reachable" +else + fix "floci down — starting via docker compose" + (cd "$ROOT" && docker compose -p floci up -d >/dev/null 2>&1) || true + for i in $(seq 1 30); do + curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && break + sleep 2 + done + curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && ok "floci back up" || warn "floci still unreachable — check 'docker compose -p floci ps'" +fi + +# 2. cluster running -------------------------------------------------------- +step "k3d cluster" +if ! k3d cluster list "$CLUSTER" >/dev/null 2>&1; then + warn "cluster '$CLUSTER' does not exist — run 'make deploy' first" + exit 1 +fi +if docker inspect -f '{{.State.Running}}' "$NODE" 2>/dev/null | grep -q true; then + ok "node container running" +else + fix "node stopped — starting cluster" + k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true +fi + +# 3. node Ready + flannel CNI ---------------------------------------------- +step "node readiness + flannel CNI" +$KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true +if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then + ok "flannel subnet.env present" +else + fix "flannel subnet.env missing — bouncing node to reinitialize CNI" + k3d cluster stop "$CLUSTER" >/dev/null 2>&1 && k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true + $KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true + if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then + ok "flannel recovered" + else + warn "flannel still missing — inspect 'docker logs $NODE'" + fi +fi + +# 4. host.k3d.internal (ESO -> floci) -------------------------------------- +step "host.k3d.internal resolution (ESO -> floci)" +fix "re-asserting host.k3d.internal (durable coredns-custom mapping)" +"$DIR/coredns-hostfix.sh" "$CLUSTER" >/dev/null 2>&1 && ok "coredns host-fix applied" || warn "coredns-hostfix.sh failed" + +# 5. webapp image on the node ---------------------------------------------- +step "webapp image on node" +"$DIR/ensure-webapp-image.sh" "$CLUSTER" || warn "could not load webapp image — see output above" + +# 6. External Secrets re-sync ---------------------------------------------- +step "External Secrets (force re-sync from floci)" +$KUBECTL -n external-secrets rollout restart deploy/external-secrets >/dev/null 2>&1 || true +$KUBECTL -n external-secrets rollout status deploy/external-secrets --timeout=90s >/dev/null 2>&1 || true +ok "ESO controller restarted" + +# 7. log shipper ------------------------------------------------------------ +step "CloudWatch log shipper" +if $KUBECTL apply -f "$ROOT/logging/cloudwatch-log-shipper.yaml" >/dev/null 2>&1; then + ok "log-shipper applied" +else + warn "log-shipper apply failed" +fi + +# 8. nudge the webapp workload --------------------------------------------- +step "webapp workload" +$KUBECTL -n webapp delete pod -l app=webapp >/dev/null 2>&1 || true +$KUBECTL -n webapp rollout status deploy/webapp --timeout=120s >/dev/null 2>&1 || true + +# 9. final status ----------------------------------------------------------- +step "status" +$KUBECTL -n webapp get pods 2>/dev/null || true +echo +$KUBECTL -n webapp get externalsecret 2>/dev/null || true +code="$(curl -s -o /dev/null -w '%{http_code}' http://localhost:30080 2>/dev/null)" +echo +if [ "$code" = "200" ]; then + ok "app responding: HTTP $code at http://localhost:30080" +else + warn "app HTTP $code (expected 200). If ExternalSecret shows SecretSyncedError," + warn "floci may have lost its secrets on reboot — reseed with 'make bootstrap'." +fi diff --git a/terraform/cluster/webapp.tf b/terraform/cluster/webapp.tf index 2966886..21846de 100644 --- a/terraform/cluster/webapp.tf +++ b/terraform/cluster/webapp.tf @@ -12,7 +12,11 @@ resource "null_resource" "ecr_image_import" { depends_on = [time_sleep.cluster_ready] + # Load the image onto the node robustly. `k3d image import` silently no-ops on + # Docker's containerd-snapshotter store (default on recent Docker/Ubuntu), so + # this helper verifies the image actually landed and falls back to a + # pull+retag if not. See scripts/ensure-webapp-image.sh. provisioner "local-exec" { - command = "k3d image import 000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27 -c webapp-test" + command = "${path.module}/../../scripts/ensure-webapp-image.sh webapp-test" } }