diff --git a/Makefile b/Makefile index 28cfb6b..0e217d7 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ SECRET_KEY ?= s3cr3t .DEFAULT_GOAL := help -.PHONY: help deploy destroy nuke bootstrap preflight cluster-up floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean +.PHONY: help deploy destroy nuke bootstrap preflight cluster-up reboot floci-up floci-down log-shipper argo-app argo-ui argo-password webapp-ui argo-pause argo-resume access status url infra secrets clean help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-16s\033[0m %s\n", $$1, $$2}' @@ -130,6 +130,9 @@ floci-down: ## Stop the floci AWS emulator preflight: ## Fail fast if Docker can't push to the floci ECR registry over HTTP @./scripts/preflight-docker-registry.sh +reboot: ## Diagnose + auto-heal the lab after a host reboot (flannel, DNS, image, ESO) + @./scripts/recover.sh + bootstrap: preflight floci-up ## Start floci + apply secrets->infra->secrets(KMS) in the required order @echo "==> floci up; applying secrets (pass 1, no KMS)" $(RETRY) 'cd $(TF_SECRETS) && $(TF) init -input=false && $(TF) apply $(APPROVE) -var="db_password=$(DB_PASSWORD)" -var="api_key=$(API_KEY)" -var="secret_key=$(SECRET_KEY)"' diff --git a/README.md b/README.md index 6e139f4..42c931e 100644 --- a/README.md +++ b/README.md @@ -488,6 +488,40 @@ helm diff upgrade webapp charts/webapp/ -n webapp --- +## After a reboot: `make reboot` + +A host reboot restarts the k3d node container, which breaks several things a +plain `kubectl get pods` won't explain — and they compound: + +| Symptom | Cause | +| --- | --- | +| Pods stuck `ContainerCreating`, `failed to setup network ... /run/flannel/subnet.env` | flannel's subnet file lives on tmpfs (`/run`), wiped on node restart | +| ESO `SecretSyncedError`, `lookup host.k3d.internal ... i/o timeout` | k3s strips `host.k3d.internal` from CoreDNS on restart | +| webapp `ImagePullBackOff`, `dial tcp 127.0.0.1:5100: connection refused` | the image dropped out of the node's containerd; the node can't pull the floci ECR ref | +| ESO/webapp errors, floci unreachable | floci container down / lost un-flushed data | + +`make reboot` diagnoses and auto-heals all of them (idempotent, safe to run +anytime — it never touches Terraform state, floci secrets, or the infra/secrets +roots): + +```bash +make reboot +``` + +It checks and fixes, in order: floci up → cluster started → **flannel** (bounces +the node if `subnet.env` is missing) → **host.k3d.internal** (durable +coredns-custom mapping) → **webapp image on the node** (see below) → ESO +re-sync → log shipper → webapp rollout, then prints status and the app HTTP code. + +> **Image loading on modern Docker.** If `docker info` shows the +> `containerd-snapshotter` image store (default on recent Docker/Ubuntu), +> `k3d image import` silently no-ops (its `docker save` output is an OCI layout +> the importer rejects: *"content digest not found"*). `make deploy` and +> `make reboot` both use `scripts/ensure-webapp-image.sh`, which verifies the +> image actually landed and falls back to pulling the identical upstream image +> into the node's containerd and retagging it. So the image step is reliable +> regardless of Docker's storage driver. + ## Key Commands ```bash diff --git a/scripts/ensure-webapp-image.sh b/scripts/ensure-webapp-image.sh new file mode 100755 index 0000000..ebf94ae --- /dev/null +++ b/scripts/ensure-webapp-image.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Ensure the webapp image is present in the k3d node's containerd, so the pod +# (imagePullPolicy: IfNotPresent) never tries to pull it. The node CANNOT pull +# the floci ECR ref at runtime — inside the node that hostname resolves to the +# node's own loopback — so the image must be loaded onto the node out-of-band. +# +# Two load paths, tried in order, because `k3d image import` silently no-ops on +# Docker's containerd-snapshotter image store (the default on recent Docker / +# Ubuntu): `docker save` there emits an OCI layout that the importer rejects with +# "content digest ... not found", yet reports success. So we verify after import +# and fall back to pulling the identical upstream image straight into the node's +# containerd (the node has internet even when the pod CNI is down) and retagging. +# +# Idempotent. Usage: ensure-webapp-image.sh [cluster-name] +set -u + +CLUSTER="${1:-webapp-test}" +NODE="k3d-${CLUSTER}-server-0" +REF="000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27" +# The ECR image is a straight retag of this upstream image (see terraform/infra/ +# ecr.tf — pull + tag + push, no build), so the content is identical. +SRC="docker.io/nginxinc/nginx-unprivileged:1.27" + +present() { docker exec "$NODE" crictl images 2>/dev/null | grep -q 'webapp/nginx'; } + +if present; then + echo "ensure-image: $REF already on $NODE" + exit 0 +fi + +# Path 1: k3d image import (works on the classic overlay2/docker image store). +echo "ensure-image: image missing — trying 'k3d image import'" +k3d image import "$REF" -c "$CLUSTER" >/dev/null 2>&1 || true +if present; then + echo "ensure-image: loaded via k3d image import" + exit 0 +fi + +# Path 2: containerd-snapshotter store breaks k3d/ctr import — pull upstream into +# the node's containerd and retag as the ECR ref the Deployment references. +echo "ensure-image: k3d import didn't land (containerd-snapshotter store?) — pulling upstream + retag" +docker exec "$NODE" crictl pull "$SRC" +docker exec "$NODE" ctr -n k8s.io images tag "$SRC" "$REF" 2>/dev/null || true +if present; then + echo "ensure-image: loaded via pull+retag" + exit 0 +fi + +echo "ensure-image: FAILED to load $REF onto $NODE" >&2 +exit 1 diff --git a/scripts/recover.sh b/scripts/recover.sh new file mode 100755 index 0000000..1d67441 --- /dev/null +++ b/scripts/recover.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +# make reboot — diagnose and auto-heal the lab after a host reboot. +# +# A reboot restarts the k3d node container, which reliably breaks several things +# that a plain `kubectl get pods` won't explain: +# 1. floci (the AWS emulator) may be down / lost un-flushed data +# 2. the k3d cluster is stopped +# 3. flannel's /run/flannel/subnet.env is on tmpfs -> wiped -> pod sandboxes +# fail ("failed to setup network ... open /run/flannel/subnet.env") +# 4. host.k3d.internal is stripped from CoreDNS -> ESO can't reach floci +# ("dial tcp: lookup host.k3d.internal ... i/o timeout") +# 5. the webapp image can drop out of the node's containerd -> ImagePullBackOff +# +# This script checks each, reports what it finds, and fixes it. Idempotent and +# safe to run anytime (not just after a reboot). It does NOT touch Terraform +# state, floci secrets, or the infra/secrets roots. +set -u + +CLUSTER="webapp-test" +CTX="k3d-${CLUSTER}" +NODE="k3d-${CLUSTER}-server-0" +KUBECTL="kubectl --context ${CTX}" +DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$DIR/.." && pwd)" + +step() { printf '\n=== %s ===\n' "$1"; } +ok() { printf ' [ok] %s\n' "$1"; } +fix() { printf ' [fix] %s\n' "$1"; } +warn() { printf ' [!] %s\n' "$1"; } + +# 1. floci ------------------------------------------------------------------ +step "floci AWS emulator (:4566)" +if curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1; then + ok "floci reachable" +else + fix "floci down — starting via docker compose" + (cd "$ROOT" && docker compose -p floci up -d >/dev/null 2>&1) || true + for i in $(seq 1 30); do + curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && break + sleep 2 + done + curl -sf http://localhost:4566/_localstack/health >/dev/null 2>&1 && ok "floci back up" || warn "floci still unreachable — check 'docker compose -p floci ps'" +fi + +# 2. cluster running -------------------------------------------------------- +step "k3d cluster" +if ! k3d cluster list "$CLUSTER" >/dev/null 2>&1; then + warn "cluster '$CLUSTER' does not exist — run 'make deploy' first" + exit 1 +fi +if docker inspect -f '{{.State.Running}}' "$NODE" 2>/dev/null | grep -q true; then + ok "node container running" +else + fix "node stopped — starting cluster" + k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true +fi + +# 3. node Ready + flannel CNI ---------------------------------------------- +step "node readiness + flannel CNI" +$KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true +if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then + ok "flannel subnet.env present" +else + fix "flannel subnet.env missing — bouncing node to reinitialize CNI" + k3d cluster stop "$CLUSTER" >/dev/null 2>&1 && k3d cluster start "$CLUSTER" >/dev/null 2>&1 || true + $KUBECTL wait --for=condition=Ready node --all --timeout=120s >/dev/null 2>&1 || true + if docker exec "$NODE" test -f /run/flannel/subnet.env 2>/dev/null; then + ok "flannel recovered" + else + warn "flannel still missing — inspect 'docker logs $NODE'" + fi +fi + +# 4. host.k3d.internal (ESO -> floci) -------------------------------------- +step "host.k3d.internal resolution (ESO -> floci)" +fix "re-asserting host.k3d.internal (durable coredns-custom mapping)" +"$DIR/coredns-hostfix.sh" "$CLUSTER" >/dev/null 2>&1 && ok "coredns host-fix applied" || warn "coredns-hostfix.sh failed" + +# 5. webapp image on the node ---------------------------------------------- +step "webapp image on node" +"$DIR/ensure-webapp-image.sh" "$CLUSTER" || warn "could not load webapp image — see output above" + +# 6. External Secrets re-sync ---------------------------------------------- +step "External Secrets (force re-sync from floci)" +$KUBECTL -n external-secrets rollout restart deploy/external-secrets >/dev/null 2>&1 || true +$KUBECTL -n external-secrets rollout status deploy/external-secrets --timeout=90s >/dev/null 2>&1 || true +ok "ESO controller restarted" + +# 7. log shipper ------------------------------------------------------------ +step "CloudWatch log shipper" +if $KUBECTL apply -f "$ROOT/logging/cloudwatch-log-shipper.yaml" >/dev/null 2>&1; then + ok "log-shipper applied" +else + warn "log-shipper apply failed" +fi + +# 8. nudge the webapp workload --------------------------------------------- +step "webapp workload" +$KUBECTL -n webapp delete pod -l app=webapp >/dev/null 2>&1 || true +$KUBECTL -n webapp rollout status deploy/webapp --timeout=120s >/dev/null 2>&1 || true + +# 9. final status ----------------------------------------------------------- +step "status" +$KUBECTL -n webapp get pods 2>/dev/null || true +echo +$KUBECTL -n webapp get externalsecret 2>/dev/null || true +code="$(curl -s -o /dev/null -w '%{http_code}' http://localhost:30080 2>/dev/null)" +echo +if [ "$code" = "200" ]; then + ok "app responding: HTTP $code at http://localhost:30080" +else + warn "app HTTP $code (expected 200). If ExternalSecret shows SecretSyncedError," + warn "floci may have lost its secrets on reboot — reseed with 'make bootstrap'." +fi diff --git a/terraform/cluster/webapp.tf b/terraform/cluster/webapp.tf index 2966886..21846de 100644 --- a/terraform/cluster/webapp.tf +++ b/terraform/cluster/webapp.tf @@ -12,7 +12,11 @@ resource "null_resource" "ecr_image_import" { depends_on = [time_sleep.cluster_ready] + # Load the image onto the node robustly. `k3d image import` silently no-ops on + # Docker's containerd-snapshotter store (default on recent Docker/Ubuntu), so + # this helper verifies the image actually landed and falls back to a + # pull+retag if not. See scripts/ensure-webapp-image.sh. provisioner "local-exec" { - command = "k3d image import 000000000000.dkr.ecr.us-east-1.localhost.localstack.cloud:5100/webapp/nginx:1.27 -c webapp-test" + command = "${path.module}/../../scripts/ensure-webapp-image.sh webapp-test" } }