diff --git a/.github/workflows/chart.yml b/.github/workflows/chart.yml index 9b13d8e..84426f3 100644 --- a/.github/workflows/chart.yml +++ b/.github/workflows/chart.yml @@ -28,6 +28,9 @@ jobs: - name: Lint chart run: helm lint charts/express-botx + - name: Test TLS chart rendering + run: bash charts/express-botx/tests/tls-smoke.sh + publish: if: startsWith(github.ref, 'refs/tags/chart-') needs: lint diff --git a/CHANGELOG.md b/CHANGELOG.md index e28749c..c962649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +### Added: HTTPS/TLS serving с hot reload сертификата + +- `serve` и `serve --enqueue` поддерживают opt-in HTTPS через YAML, env или + `--tls-cert`/`--tls-key`; минимальная версия — TLS 1.2. +- Сертификат и ключ перечитываются по content hash без остановки listener; ошибка + ротации не сбрасывает последнюю корректную пару. +- Helm умеет создать cert-manager Certificate или смонтировать существующий TLS + Secret, переключает probes/backend port name на HTTPS и сохраняет прежний HTTP + render при `tls.enabled=false`. + ## 0.34.0 ### ⚠️ Breaking: GitLab-конфигурация только через senders diff --git a/README.md b/README.md index 4e80131..3b46764 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,22 @@ curl -X POST http://localhost:8080/api/v1/send \ -d '{"message": "Test from express-botx web api"}' ``` +TLS включается, когда после объединения YAML, переменных окружения и CLI заданы +оба пути. HTTPS заменяет HTTP на том же `--listen` и действует как для обычного +`serve`, так и для `serve --enqueue`: + +```bash +express-botx serve \ + --listen :8443 \ + --tls-cert /etc/express-botx/tls/tls.crt \ + --tls-key /etc/express-botx/tls/tls.key + +curl --cacert ca.crt https://localhost:8443/healthz +``` + +Сертификат перечитывается без перезапуска. При ошибке ротации сервер продолжает +отдавать последнюю корректную пару. + Эндпоинты (все POST требуют `Authorization: Bearer `): diff --git a/charts/express-botx/README.md b/charts/express-botx/README.md index a6375bc..22f8388 100644 --- a/charts/express-botx/README.md +++ b/charts/express-botx/README.md @@ -47,6 +47,7 @@ config: |----------|----------|--------------| | `mode` | Режим: `serve`, `serve-enqueue`, `worker` | `serve` | | `replicaCount` | Количество реплик | `1` | +| `containerPort` | Порт контейнера и probes | `8080` | | `image.repository` | Docker-образ | `lavr/express-botx` | | `image.tag` | Тег образа | `-rootless` из Chart.yaml | | `image.pullPolicy` | Pull policy | `IfNotPresent` | @@ -76,8 +77,19 @@ config: | `config.catalog.publish` | Worker публикует catalog | `true` | | `config.catalog.publish_interval` | Интервал публикации | `30s` | | `existingSecret` | Имя существующего Secret с `config.yaml` | `""` | +| `tls.enabled` | Включить HTTPS на pod | `false` | +| `tls.mountPath` | Каталог TLS Secret | `/etc/express-botx/tls` | +| `tls.reloadInterval` | Интервал hot reload | `60s` | +| `tls.certManager.enabled` | Создать Certificate | `false` | +| `tls.certManager.issuerRef.name` | Имя issuer | `""` | +| `tls.certManager.issuerRef.kind` | Issuer kind | `ClusterIssuer` | +| `tls.certManager.dnsNames` | SAN или fallback из Ingress hosts | `[]` | +| `tls.certManager.duration` | Опциональная duration | `""` | +| `tls.certManager.renewBefore` | Опциональная renewBefore | `""` | +| `tls.existingSecret` | Secret с `tls.crt`/`tls.key` | `""` | | `service.type` | Тип сервиса | `ClusterIP` | | `service.port` | Порт сервиса | `80` | +| `service.targetPort` | Независимый numeric/named targetPort | `8080` | | `ingress.enabled` | Включить Ingress | `false` | | `ingress.className` | Ingress class | `""` | | `ingress.hosts` | Список хостов | `[]` | @@ -88,6 +100,73 @@ config: | `autoscaling.enabled` | Включить HPA | `false` | | `extraEnv` | Дополнительные переменные окружения | `[]` | +### HTTPS/TLS + +cert-manager: + +```yaml +containerPort: 8443 +service: + targetPort: 8443 +config: + server: + listen: ":8443" +tls: + enabled: true + certManager: + enabled: true + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: [botx.example.com] +``` + +Существующий Secret с обязательными ключами `tls.crt` и `tls.key`: + +```bash +kubectl create secret tls express-botx-tls --cert=cert.pem --key=key.pem +``` + +Имена `cert.pem` и `key.pem` здесь относятся только к локальным файлам: команда +`kubectl create secret tls` создаёт стандартные ключи `tls.crt` и `tls.key`. + +```yaml +containerPort: 8443 +service: + targetPort: 8443 +config: + server: + listen: ":8443" +tls: + enabled: true + existingSecret: express-botx-tls +``` + +Для ingress-nginx при pod-level TLS: + +```yaml +ingress: + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" +``` + +Chart не добавляет controller-specific аннотации автоматически. `ingress.tls` +настраивает клиент → Ingress, а `tls.enabled` — Ingress/kubelet → pod. При TLS +`service.targetPort` должен совпадать с `containerPort` или именем порта `https`, +а приложение должно слушать тот же numeric port. + +Chart не подменяет `EXPRESS_BOTX_SERVER_LISTEN`: оператор явно согласует +`containerPort`, `service.targetPort` и listen из structured config, `configRaw`, +внешнего config Secret или `extraEnv`. `service.targetPort` сохраняет буквальный +default `8080`, чтобы старые `tpl`-выражения продолжали работать. TLS запрещён +для `mode: worker`; `tls.certManager.enabled` и `tls.existingSecret` +взаимоисключающие. + +Произвольные имена ключей TLS Secret не переназначаются. Уже существующий Secret, +ключи которого сами называются `cert.pem`/`key.pem`, не поддерживается: при +отсутствии `tls.crt`/`tls.key` kubelet не смонтирует volume и pod не запустит +приложение. + ### Deployment patterns Для async-режима рекомендуется два отдельных Deployment: API-сервер (`serve --enqueue`) и worker. diff --git a/charts/express-botx/templates/NOTES.txt b/charts/express-botx/templates/NOTES.txt index ed1ae60..c53a78d 100644 --- a/charts/express-botx/templates/NOTES.txt +++ b/charts/express-botx/templates/NOTES.txt @@ -10,7 +10,7 @@ To access the API, run: kubectl port-forward svc/{{ include "express-botx.fullname" . }} {{ .Values.service.port }}:{{ .Values.service.targetPort }} Then send a request: - curl http://localhost:{{ .Values.service.port }}/healthz + curl {{ if .Values.tls.enabled }}-k https{{ else }}http{{ end }}://localhost:{{ .Values.service.port }}/healthz {{- end }} Configured bots: diff --git a/charts/express-botx/templates/_helpers.tpl b/charts/express-botx/templates/_helpers.tpl index 1a35ce6..04765cc 100644 --- a/charts/express-botx/templates/_helpers.tpl +++ b/charts/express-botx/templates/_helpers.tpl @@ -65,3 +65,78 @@ Config secret name. {{- define "express-botx.secretName" -}} {{- default (include "express-botx.fullname" .) .Values.existingSecret }} {{- end }} + +{{- define "express-botx.portName" -}} +{{- if .Values.tls.enabled }}https{{ else }}http{{ end }} +{{- end }} + +{{- define "express-botx.tls.secretName" -}} +{{- if .Values.tls.existingSecret -}} +{{- .Values.tls.existingSecret -}} +{{- else -}} +{{- printf "%s-tls" (include "express-botx.fullname" .) -}} +{{- end -}} +{{- end }} + +{{- define "express-botx.tls.dnsNames" -}} +{{- $names := list -}} +{{- range .Values.tls.certManager.dnsNames -}} + {{- $name := trim (toString (default "" .)) -}} + {{- if $name }}{{- $names = append $names $name -}}{{- end -}} +{{- end -}} +{{- if and (eq (len $names) 0) .Values.ingress.enabled -}} + {{- range .Values.ingress.hosts -}} + {{- $hostConfig := default dict . -}} + {{- $host := trim (toString (default "" $hostConfig.host)) -}} + {{- if $host }}{{- $names = append $names $host -}}{{- end -}} + {{- end -}} +{{- end -}} +{{- if eq (len $names) 0 -}} + {{- fail "tls.certManager.dnsNames is required (or enable ingress with non-empty hosts)" -}} +{{- end -}} +{{- range $names }} +- {{ . | quote }} +{{- end -}} +{{- end }} + +{{- define "express-botx.tls.validate" -}} +{{- if .Values.tls.enabled -}} + {{- if eq .Values.mode "worker" }}{{- fail "tls.enabled is not supported with mode=worker" -}}{{- end -}} + {{- $cm := .Values.tls.certManager.enabled -}} + {{- $tlsSecret := trim (toString (default "" .Values.tls.existingSecret)) -}} + {{- $issuerRef := default dict .Values.tls.certManager.issuerRef -}} + {{- if and $cm $tlsSecret }}{{- fail "tls.certManager.enabled and tls.existingSecret are mutually exclusive" -}}{{- end -}} + {{- if and (not $cm) (not $tlsSecret) }}{{- fail "tls.enabled requires tls.certManager.enabled or tls.existingSecret" -}}{{- end -}} + {{- if and $cm (not (trim (toString (default "" $issuerRef.name)))) }}{{- fail "tls.certManager.issuerRef.name is required" -}}{{- end -}} + + {{- $reloadInterval := trim (toString (default "" .Values.tls.reloadInterval)) -}} + {{- $durationPattern := "^[+]?(([0-9]+([.][0-9]*)?|[.][0-9]+)(ns|us|µs|μs|ms|s|m|h))+$" -}} + {{- if or (not (regexMatch $durationPattern $reloadInterval)) (not (regexMatch "[1-9]" $reloadInterval)) -}} + {{- fail (printf "tls.reloadInterval must be a positive Go duration (for example 60s), got %q" $reloadInterval) -}} + {{- end -}} + + {{- $container := toString .Values.containerPort -}} + {{- $target := toString .Values.service.targetPort -}} + {{- $portName := include "express-botx.portName" . -}} + {{- if and (ne $target $container) (ne $target $portName) -}} + {{- fail (printf "service.targetPort must equal containerPort (%s) or port name %q when TLS is enabled" $container $portName) -}} + {{- end -}} + + {{- $listenEnv := false -}} + {{- range .Values.extraEnv -}} + {{- if eq (default "" .name) "EXPRESS_BOTX_SERVER_LISTEN" }}{{- $listenEnv = true -}}{{- end -}} + {{- end -}} + {{- if and (not .Values.configRaw) (not .Values.existingSecret) (not $listenEnv) -}} + {{- $config := default dict .Values.config -}} + {{- $server := default dict $config.server -}} + {{- $listen := toString (default "" $server.listen) -}} + {{- $suffix := regexFind ":[0-9]+$" $listen -}} + {{- if $suffix -}} + {{- $listenPort := trimPrefix ":" $suffix -}} + {{- if ne $listenPort $container -}} + {{- fail (printf "config.server.listen port (%s) must equal containerPort (%s) when TLS is enabled" $listenPort $container) -}} + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- end }} diff --git a/charts/express-botx/templates/certificate.yaml b/charts/express-botx/templates/certificate.yaml new file mode 100644 index 0000000..72a4ef0 --- /dev/null +++ b/charts/express-botx/templates/certificate.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.tls.enabled .Values.tls.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ printf "%s-tls" (include "express-botx.fullname" .) }} + labels: + {{- include "express-botx.labels" . | nindent 4 }} +spec: + secretName: {{ printf "%s-tls" (include "express-botx.fullname" .) }} + dnsNames: + {{- include "express-botx.tls.dnsNames" . | nindent 4 }} + issuerRef: + name: {{ .Values.tls.certManager.issuerRef.name | quote }} + kind: {{ .Values.tls.certManager.issuerRef.kind | quote }} + {{- with .Values.tls.certManager.duration }} + duration: {{ . | quote }} + {{- end }} + {{- with .Values.tls.certManager.renewBefore }} + renewBefore: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/charts/express-botx/templates/deployment.yaml b/charts/express-botx/templates/deployment.yaml index df88fa3..49588f6 100644 --- a/charts/express-botx/templates/deployment.yaml +++ b/charts/express-botx/templates/deployment.yaml @@ -1,3 +1,4 @@ +{{- include "express-botx.tls.validate" . -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -50,8 +51,8 @@ spec: {{- end }} ports: {{- if ne .Values.mode "worker" }} - - name: http - containerPort: 8080 + - name: {{ include "express-botx.portName" . }} + containerPort: {{ .Values.containerPort }} protocol: TCP {{- end }} {{- if and (eq .Values.mode "worker") .Values.config.worker }} @@ -80,13 +81,19 @@ spec: livenessProbe: httpGet: path: /healthz - port: http + port: {{ include "express-botx.portName" . }} + {{- if .Values.tls.enabled }} + scheme: HTTPS + {{- end }} initialDelaySeconds: 5 periodSeconds: 10 readinessProbe: httpGet: path: /healthz - port: http + port: {{ include "express-botx.portName" . }} + {{- if .Values.tls.enabled }} + scheme: HTTPS + {{- end }} initialDelaySeconds: 2 periodSeconds: 5 {{- end }} @@ -101,9 +108,22 @@ spec: readOnly: true - name: cache mountPath: /tmp/express-botx + {{- if .Values.tls.enabled }} + - name: tls + mountPath: {{ .Values.tls.mountPath | quote }} + readOnly: true + {{- end }} env: - name: EXPRESS_BOTX_CONFIG value: /etc/express-botx/config.yaml + {{- if .Values.tls.enabled }} + - name: EXPRESS_BOTX_SERVER_TLS_CERT + value: {{ printf "%s/tls.crt" (trimSuffix "/" .Values.tls.mountPath) | quote }} + - name: EXPRESS_BOTX_SERVER_TLS_KEY + value: {{ printf "%s/tls.key" (trimSuffix "/" .Values.tls.mountPath) | quote }} + - name: EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL + value: {{ .Values.tls.reloadInterval | quote }} + {{- end }} {{- with .Values.extraEnv }} {{- tpl (toYaml .) $ | nindent 12 }} {{- end }} @@ -116,6 +136,16 @@ spec: path: config.yaml - name: cache emptyDir: {} + {{- if .Values.tls.enabled }} + - name: tls + secret: + secretName: {{ include "express-botx.tls.secretName" . | quote }} + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/express-botx/templates/ingress.yaml b/charts/express-botx/templates/ingress.yaml index dfa7c46..df4f020 100644 --- a/charts/express-botx/templates/ingress.yaml +++ b/charts/express-botx/templates/ingress.yaml @@ -35,7 +35,7 @@ spec: service: name: {{ include "express-botx.fullname" $ }} port: - name: http + name: {{ include "express-botx.portName" $ }} {{- end }} {{- end }} {{- end }} diff --git a/charts/express-botx/templates/service.yaml b/charts/express-botx/templates/service.yaml index c2335f7..a88783e 100644 --- a/charts/express-botx/templates/service.yaml +++ b/charts/express-botx/templates/service.yaml @@ -10,6 +10,6 @@ spec: - port: {{ .Values.service.port }} targetPort: {{ .Values.service.targetPort }} protocol: TCP - name: http + name: {{ include "express-botx.portName" . }} selector: {{- include "express-botx.selectorLabels" . | nindent 4 }} diff --git a/charts/express-botx/tests/tls-smoke.sh b/charts/express-botx/tests/tls-smoke.sh new file mode 100755 index 0000000..e2b8082 --- /dev/null +++ b/charts/express-botx/tests/tls-smoke.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +render() { helm template test "$chart_dir" "$@"; } +render_with_notes() { helm install test "$chart_dir" --dry-run=client "$@"; } +expect_fail() { + local name="$1" + shift + if render "$@" >/dev/null 2>&1; then + echo "expected failure: $name" >&2 + exit 1 + fi +} +expect_fail_with() { + local name="$1" + local message="$2" + shift 2 + local output + if output="$(render "$@" 2>&1)"; then + echo "expected failure: $name" >&2 + exit 1 + fi + if ! grep -Fq "$message" <<<"$output"; then + echo "unexpected failure: $name" >&2 + echo "$output" >&2 + exit 1 + fi +} + +assert() { grep -Fq -- "$2" <<<"$1" || { echo "missing: $2" >&2; exit 1; }; } +refute() { ! grep -Fq -- "$2" <<<"$1" || { echo "unexpected: $2" >&2; exit 1; }; } +assert_literal_value() { + grep -Eq "value: ['\"]?$2['\"]?$" <<<"$1" || { + echo "missing literal value: $2" >&2 + exit 1 + } +} + +default="$(render_with_notes)" +refute "$default" 'kind: Certificate' +refute "$default" 'EXPRESS_BOTX_SERVER_TLS_CERT' +refute "$default" 'scheme: HTTPS' +assert "$default" 'name: http' +assert "$default" 'containerPort: 8080' +assert "$default" 'targetPort: 8080' +assert "$default" 'curl http://localhost:80/healthz' +if [[ "$(grep -Fc 'port: http' <<<"$default")" -lt 2 ]]; then + echo "both HTTP probes must use the named port" >&2 + exit 1 +fi + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +cat >"$tmp/legacy.yaml" <<'YAML' +configRaw: | + server: + listen: ":{{ .Values.service.targetPort }}" +extraEnv: + - name: PORT_COPY + value: "{{ .Values.service.targetPort }}" +YAML +legacy="$(helm template test "$chart_dir" -f "$tmp/legacy.yaml")" +assert "$legacy" 'listen: ":8080"' +assert_literal_value "$legacy" '8080' +refute "$legacy" 'EXPRESS_BOTX_SERVER_LISTEN' + +cat >"$tmp/tls.yaml" <<'YAML' +containerPort: 8443 +service: + targetPort: 8443 +config: + server: + listen: ":8443" +tls: + enabled: true + existingSecret: provided-tls + reloadInterval: 15s +ingress: + enabled: true + hosts: + - host: api.example.com + paths: + - path: / + pathType: Prefix +YAML +secure="$(render_with_notes -f "$tmp/tls.yaml")" +refute "$secure" 'kind: Certificate' +assert "$secure" 'name: https' +assert "$secure" 'containerPort: 8443' +assert "$secure" 'targetPort: 8443' +assert "$secure" 'port: https' +assert "$secure" 'scheme: HTTPS' +assert "$secure" 'secretName: "provided-tls"' +assert "$secure" 'key: tls.crt' +assert "$secure" 'path: tls.crt' +assert "$secure" 'key: tls.key' +assert "$secure" 'path: tls.key' +assert "$secure" 'mountPath: "/etc/express-botx/tls"' +assert "$secure" 'readOnly: true' +assert "$secure" 'EXPRESS_BOTX_SERVER_TLS_CERT' +assert "$secure" 'value: "/etc/express-botx/tls/tls.crt"' +assert "$secure" 'EXPRESS_BOTX_SERVER_TLS_KEY' +assert "$secure" 'value: "/etc/express-botx/tls/tls.key"' +assert "$secure" 'EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL' +assert "$secure" 'value: "15s"' +refute "$secure" 'checksum/tls' +if [[ "$(grep -Fc 'EXPRESS_BOTX_SERVER_TLS_' <<<"$secure")" -ne 3 ]]; then + echo "TLS must inject exactly three server environment variables" >&2 + exit 1 +fi +if [[ "$(grep -Fc 'port: https' <<<"$secure")" -lt 2 ]]; then + echo "both HTTPS probes must use the named port" >&2 + exit 1 +fi +if [[ "$(grep -Fc 'name: https' <<<"$secure")" -lt 3 ]]; then + echo "container, Service, and Ingress must use the HTTPS port name" >&2 + exit 1 +fi + +secure_notes="$(render_with_notes -f "$tmp/tls.yaml" --set ingress.enabled=false)" +assert "$secure_notes" 'kubectl port-forward svc/test-express-botx 80:8443' +assert "$secure_notes" 'curl -k https://localhost:80/healthz' + +certificate="$(render --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=123 \ + --set-string 'tls.certManager.dnsNames[0]=*.example.com')" +assert "$certificate" 'kind: Certificate' +assert "$certificate" 'name: "123"' +assert "$certificate" '- "*.example.com"' +assert "$certificate" 'EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL' + +autofill="$(render --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=issuer --set ingress.enabled=true \ + --set-string ingress.hosts[0].host=api.example.com \ + --set-string ingress.hosts[0].paths[0].path=/ \ + --set-string ingress.hosts[0].paths[0].pathType=Prefix)" +assert "$autofill" '- "api.example.com"' + +named="$(render --set tls.enabled=true --set-string tls.existingSecret=provided-tls \ + --set-string service.targetPort=https)" +assert "$named" 'targetPort: https' + +cat >"$tmp/raw.yaml" <<'YAML' +containerPort: 8443 +service: {targetPort: 8443} +configRaw: | + server: + listen: ":9443" +tls: {enabled: true, existingSecret: provided-tls} +YAML +render -f "$tmp/raw.yaml" >/dev/null + +cat >"$tmp/config-secret.yaml" <<'YAML' +containerPort: 8443 +service: {targetPort: 8443} +existingSecret: provided-config +tls: {enabled: true, existingSecret: provided-tls} +YAML +render -f "$tmp/config-secret.yaml" >/dev/null + +cat >"$tmp/env.yaml" <<'YAML' +containerPort: 8443 +service: {targetPort: 8443} +extraEnv: + - name: EXPRESS_BOTX_SERVER_LISTEN + value: ":9443" +tls: {enabled: true, existingSecret: provided-tls} +YAML +render -f "$tmp/env.yaml" >/dev/null + +server_null="$(render --set tls.enabled=true \ + --set-string tls.existingSecret=provided-tls --set-json config.server=null)" +assert "$server_null" 'kind: Deployment' +assert "$server_null" 'containerPort: 8080' + +render --set tls.enabled=true --set-string tls.existingSecret=provided-tls \ + --set-string tls.reloadInterval=.5s >/dev/null + +expect_fail "missing source" --set tls.enabled=true +expect_fail_with "invalid reload interval" \ + "tls.reloadInterval must be a positive Go duration" \ + --set tls.enabled=true --set-string tls.existingSecret=provided-tls \ + --set-string tls.reloadInterval=soon +expect_fail_with "zero reload interval" \ + "tls.reloadInterval must be a positive Go duration" \ + --set tls.enabled=true --set-string tls.existingSecret=provided-tls \ + --set-string tls.reloadInterval=0s +expect_fail_with "negative reload interval" \ + "tls.reloadInterval must be a positive Go duration" \ + --set tls.enabled=true --set-string tls.existingSecret=provided-tls \ + --set-string tls.reloadInterval=-1s +expect_fail_with "null TLS Secret source" \ + "tls.enabled requires tls.certManager.enabled or tls.existingSecret" \ + --set tls.enabled=true --set-json tls.existingSecret=null +expect_fail "both sources" --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=issuer \ + --set-string tls.certManager.dnsNames[0]=api.example.com \ + --set-string tls.existingSecret=provided-tls +expect_fail "missing issuer" --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.dnsNames[0]=api.example.com +expect_fail_with "null issuer" "tls.certManager.issuerRef.name is required" \ + --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-json tls.certManager.issuerRef.name=null \ + --set-string tls.certManager.dnsNames[0]=api.example.com +expect_fail_with "null issuerRef mapping" \ + "tls.certManager.issuerRef.name is required" \ + --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-json tls.certManager.issuerRef=null \ + --set-string tls.certManager.dnsNames[0]=api.example.com +expect_fail "worker" --set mode=worker --set tls.enabled=true \ + --set-string tls.existingSecret=provided-tls +expect_fail "empty DNS" --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=issuer --set ingress.enabled=true +expect_fail_with "null DNS" \ + "tls.certManager.dnsNames is required (or enable ingress with non-empty hosts)" \ + --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=issuer \ + --set-json 'tls.certManager.dnsNames=[null]' +expect_fail_with "null ingress DNS" \ + "tls.certManager.dnsNames is required (or enable ingress with non-empty hosts)" \ + --set tls.enabled=true --set tls.certManager.enabled=true \ + --set-string tls.certManager.issuerRef.name=issuer --set ingress.enabled=true \ + --set-json 'ingress.hosts=[{"host":null,"paths":[{"path":"/","pathType":"Prefix"}]}]' +expect_fail "numeric target mismatch" --set tls.enabled=true \ + --set-string tls.existingSecret=provided-tls --set containerPort=8443 +expect_fail "named target mismatch" --set tls.enabled=true \ + --set-string tls.existingSecret=provided-tls --set-string service.targetPort=wrong +expect_fail "structured listen mismatch" --set tls.enabled=true \ + --set-string tls.existingSecret=provided-tls --set containerPort=8443 \ + --set service.targetPort=8443 diff --git a/charts/express-botx/values.yaml b/charts/express-botx/values.yaml index 336a55a..6e9b98e 100644 --- a/charts/express-botx/values.yaml +++ b/charts/express-botx/values.yaml @@ -6,6 +6,8 @@ mode: serve replicaCount: 1 +containerPort: 8080 + image: repository: lavr/express-botx # Defaults to -rootless (scratch-based, distroless image). @@ -84,6 +86,22 @@ config: # The Secret must contain a "config.yaml" key with the full YAML config. existingSecret: "" +tls: + enabled: false + mountPath: /etc/express-botx/tls + # Go duration; must be positive when TLS is enabled. + reloadInterval: 60s + certManager: + enabled: false + issuerRef: + name: "" + kind: ClusterIssuer + dnsNames: [] + duration: "" + renewBefore: "" + # Existing Secret with exact keys tls.crt and tls.key. + existingSecret: "" + service: type: ClusterIP port: 80 @@ -92,6 +110,8 @@ service: ingress: enabled: false className: "" + # With tls.enabled, configure the selected Ingress controller to connect to + # the Service backend over HTTPS. See docs/deployment.md for an example. annotations: {} hosts: [] # - host: express-botx.example.com diff --git a/docs/configuration.md b/docs/configuration.md index 01e0849..229f58d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -110,6 +110,37 @@ multi-bot-конфигурации каждая цель доставки дол `bot`; raw UUID и алиас без bot binding отклоняются на старте. Подробнее — в [docs/integrations.md](integrations.md#изоляция-отправителей-и-скоуп-чатов). +### HTTPS/TLS для `serve` + +```yaml +server: + listen: ":8443" + tls: + cert_file: /etc/express-botx/tls/tls.crt + key_file: /etc/express-botx/tls/tls.key + reload_interval: 60s +``` + +TLS действует и для прямого `serve`, и для `serve --enqueue`; HTTPS заменяет +HTTP на том же адресе `server.listen`. Настройки объединяются с приоритетом +YAML → env → CLI: YAML-поля `cert_file`/`key_file`, переменные +`EXPRESS_BOTX_SERVER_TLS_CERT`/`EXPRESS_BOTX_SERVER_TLS_KEY` и флаги +`--tls-cert`/`--tls-key`. После объединения нужны оба пути; ровно один путь +является ошибкой. `reload_interval` можно переопределить через +`EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL`; интервал по умолчанию — `60s`, и он +должен быть положительным. + +Пустая YAML-секция `server.tls` или секция только с `reload_interval` не включает +TLS: в обоих режимах сервер запускается по HTTP и пишет один startup warning о +plaintext fallback. Если пару дополнили env или CLI, warning не выводится; итоговая +частичная пара остаётся ошибкой. Когда секции `server.tls` нет и пути не заданы +другими слоями, сервер запускается по HTTP без warning. + +Ротация использует best-effort стабильный снимок пары и сходится примерно за +один `reload_interval`; это не транзакционная атомарность двух файлов. При +временно смешанной, нечитаемой или некорректной паре сервер продолжает отдавать +последний корректный сертификат. + ## Переменные окружения | Переменная | Описание | @@ -125,6 +156,9 @@ multi-bot-конфигурации каждая цель доставки дол | `EXPRESS_BOTX_SERVER_LISTEN` | Адрес для прослушивания (serve) | | `EXPRESS_BOTX_SERVER_BASE_PATH` | Базовый путь (serve) | | `EXPRESS_BOTX_SERVER_API_KEY` | API-ключ (serve) | +| `EXPRESS_BOTX_SERVER_TLS_CERT` | Путь к PEM-сертификату | +| `EXPRESS_BOTX_SERVER_TLS_KEY` | Путь к PEM-приватному ключу | +| `EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL` | Интервал проверки, default `60s` | | `EXPRESS_BOTX_VERBOSE` | Уровень логирования: 1-3 | ## Аутентификация diff --git a/docs/deployment.md b/docs/deployment.md index df60705..0cd97aa 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -201,6 +201,78 @@ ingress: Для использования существующего секрета: `existingSecret: my-secret`. +### HTTPS до pod + +`ingress.tls` настраивает клиент → Ingress, а `tls.enabled` — HTTPS для +Ingress и kubelet probes → pod. При pod-level TLS HTTPS заменяет HTTP на том же +адресе `config.server.listen`; отдельный listener не создаётся. Нужно согласовать +`containerPort`, `service.targetPort` и фактический порт `config.server.listen`. + +Пример с Certificate от cert-manager: + +```yaml +containerPort: 8443 +service: + targetPort: 8443 +config: + server: + listen: ":8443" +tls: + enabled: true + reloadInterval: 60s + certManager: + enabled: true + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - botx.example.com +``` + +Вместо cert-manager можно использовать существующий Secret. В нём обязательны +точные ключи `tls.crt` и `tls.key`: + +```bash +kubectl create secret tls express-botx-tls \ + --cert=cert.pem \ + --key=key.pem +``` + +Флаги `--cert` и `--key` принимают произвольные локальные имена файлов; команда +выше всё равно создаёт в Secret стандартные ключи `tls.crt` и `tls.key`. + +```yaml +containerPort: 8443 +service: + targetPort: 8443 +config: + server: + listen: ":8443" +tls: + enabled: true + existingSecret: express-botx-tls +``` + +Уже существующий Secret, ключи которого сами называются `cert.pem`/`key.pem`, +не поддерживается: kubelet не сможет смонтировать указанные chart-ом +`tls.crt`/`tls.key`, поэтому контейнер не стартует. Helm не может проверить +содержимое внешнего Secret при offline-render. Источники cert-manager и +`tls.existingSecret` взаимоисключающие, а `tls.enabled` запрещён для +`mode: worker`. + +Если включён Ingress, его controller должен подключаться к Service по HTTPS. +Chart не добавляет controller-specific аннотации автоматически. Например, +для ingress-nginx: + +```yaml +ingress: + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" +``` + +Эта backend-аннотация не заменяет `ingress.tls`: тот отдельно настраивает TLS +на участке клиент → Ingress. + ### Дополнительные переменные окружения Через `extraEnv` можно передавать переменные окружения в контейнер. Значения поддерживают Go-шаблоны Helm: diff --git a/internal/cmd/serve.go b/internal/cmd/serve.go index d1f7e26..e94cc5d 100644 --- a/internal/cmd/serve.go +++ b/internal/cmd/serve.go @@ -28,18 +28,69 @@ import ( "github.com/lavr/express-botx/internal/token" ) +func resolveTLS(yamlCfg *config.TLSYAMLConfig, envCert, envKey, envInterval, flagCert, flagKey string) (*server.TLSConfig, error) { + var certFile, keyFile, intervalValue string + if yamlCfg != nil { + certFile, keyFile, intervalValue = yamlCfg.CertFile, yamlCfg.KeyFile, yamlCfg.ReloadInterval + } + if envCert != "" { + certFile = envCert + } + if envKey != "" { + keyFile = envKey + } + if envInterval != "" { + intervalValue = envInterval + } + if flagCert != "" { + certFile = flagCert + } + if flagKey != "" { + keyFile = flagKey + } + if certFile == "" && keyFile == "" { + return nil, nil + } + if certFile == "" || keyFile == "" { + return nil, fmt.Errorf("TLS requires both certificate and key paths") + } + interval := 60 * time.Second + if intervalValue != "" { + parsed, err := time.ParseDuration(intervalValue) + if err != nil { + return nil, fmt.Errorf("invalid TLS reload interval %q: %w", intervalValue, err) + } + if parsed <= 0 { + return nil, fmt.Errorf("TLS reload interval must be positive, got %s", parsed) + } + interval = parsed + } + return &server.TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: interval}, nil +} + +func warnDisabledTLS(yamlCfg *config.TLSYAMLConfig, tlsCfg *server.TLSConfig, logf func(string, ...any)) { + if yamlCfg == nil || tlsCfg != nil { + return + } + logf("serve: server.tls is present but no TLS certificate/key paths were resolved; TLS is disabled, serving plaintext HTTP") +} + func runServe(args []string, deps Deps) error { fs := flag.NewFlagSet("serve", flag.ContinueOnError) fs.SetOutput(deps.Stderr) var flags config.Flags var listenFlag string var apiKeyFlag string + var tlsCertFlag string + var tlsKeyFlag string var failFast bool var enqueueMode bool globalFlags(fs, &flags) fs.StringVar(&listenFlag, "listen", "", "address to listen on (overrides config)") fs.StringVar(&apiKeyFlag, "api-key", "", "API key for quick start (overrides config)") + fs.StringVar(&tlsCertFlag, "tls-cert", "", "path to TLS certificate file (PEM)") + fs.StringVar(&tlsKeyFlag, "tls-key", "", "path to TLS private key file (PEM)") fs.BoolVar(&failFast, "fail-fast", false, "exit if bot authentication fails at startup") fs.BoolVar(&enqueueMode, "enqueue", false, "async mode: publish to queue instead of sending directly") fs.Usage = func() { @@ -47,6 +98,10 @@ func runServe(args []string, deps Deps) error { Start an HTTP server for sending messages via API. +TLS options: + --tls-cert FILE Path to TLS certificate file (PEM) + --tls-key FILE Path to TLS private key file (PEM) + Options: `) fs.PrintDefaults() @@ -60,7 +115,7 @@ Options: } if enqueueMode { - return runServeEnqueue(flags, listenFlag, apiKeyFlag, deps) + return runServeEnqueue(flags, listenFlag, apiKeyFlag, tlsCertFlag, tlsKeyFlag, deps) } if flags.Secret != "" && flags.Token != "" { @@ -98,6 +153,19 @@ Options: if srvCfg.BasePath == "" { srvCfg.BasePath = "/api/v1" } + tlsCfg, err := resolveTLS( + cfg.Server.TLS, + os.Getenv("EXPRESS_BOTX_SERVER_TLS_CERT"), + os.Getenv("EXPRESS_BOTX_SERVER_TLS_KEY"), + os.Getenv("EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL"), + tlsCertFlag, + tlsKeyFlag, + ) + if err != nil { + return err + } + srvCfg.TLS = tlsCfg + warnDisabledTLS(cfg.Server.TLS, tlsCfg, vlog.Info) // External URL for OpenAPI docs srvCfg.ExternalURL = cfg.Server.ExternalURL @@ -770,7 +838,7 @@ func runtimeChatEntries(cfg *config.Config) []config.ChatEntry { // runServeEnqueue starts the HTTP server in async/enqueue mode. // Instead of sending directly to BotX API, requests are published to a work queue. -func runServeEnqueue(flags config.Flags, listenFlag, apiKeyFlag string, deps Deps) error { +func runServeEnqueue(flags config.Flags, listenFlag, apiKeyFlag, tlsCertFlag, tlsKeyFlag string, deps Deps) error { cfg, err := config.LoadForServeEnqueue(flags) if err != nil { return err @@ -809,6 +877,19 @@ func runServeEnqueue(flags config.Flags, listenFlag, apiKeyFlag string, deps Dep if srvCfg.BasePath == "" { srvCfg.BasePath = "/api/v1" } + tlsCfg, err := resolveTLS( + cfg.Server.TLS, + os.Getenv("EXPRESS_BOTX_SERVER_TLS_CERT"), + os.Getenv("EXPRESS_BOTX_SERVER_TLS_KEY"), + os.Getenv("EXPRESS_BOTX_SERVER_TLS_RELOAD_INTERVAL"), + tlsCertFlag, + tlsKeyFlag, + ) + if err != nil { + return err + } + srvCfg.TLS = tlsCfg + warnDisabledTLS(cfg.Server.TLS, tlsCfg, vlog.Info) srvCfg.ExternalURL = cfg.Server.ExternalURL if v := os.Getenv("EXPRESS_BOTX_SERVER_EXTERNAL_URL"); v != "" { diff --git a/internal/cmd/serve_tls_test.go b/internal/cmd/serve_tls_test.go new file mode 100644 index 0000000..1733972 --- /dev/null +++ b/internal/cmd/serve_tls_test.go @@ -0,0 +1,103 @@ +package cmd + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/lavr/express-botx/internal/config" + "github.com/lavr/express-botx/internal/server" +) + +func TestResolveTLS(t *testing.T) { + tests := []struct { + name string + yaml *config.TLSYAMLConfig + envCert, envKey, envInterval string + flagCert, flagKey string + wantCert, wantKey string + wantInterval time.Duration + wantNil, wantErr bool + }{ + {name: "absent", wantNil: true}, + {name: "empty", yaml: &config.TLSYAMLConfig{}, wantNil: true}, + {name: "interval only", yaml: &config.TLSYAMLConfig{ReloadInterval: "bad"}, wantNil: true}, + {name: "yaml default", yaml: &config.TLSYAMLConfig{CertFile: "y.crt", KeyFile: "y.key"}, wantCert: "y.crt", wantKey: "y.key", wantInterval: time.Minute}, + {name: "env overrides", yaml: &config.TLSYAMLConfig{CertFile: "y.crt", KeyFile: "y.key", ReloadInterval: "10s"}, envCert: "e.crt", envKey: "e.key", envInterval: "20s", wantCert: "e.crt", wantKey: "e.key", wantInterval: 20 * time.Second}, + {name: "flags override", envCert: "e.crt", envKey: "e.key", flagCert: "f.crt", flagKey: "f.key", wantCert: "f.crt", wantKey: "f.key", wantInterval: time.Minute}, + {name: "split layers", yaml: &config.TLSYAMLConfig{CertFile: "y.crt"}, envKey: "e.key", wantCert: "y.crt", wantKey: "e.key", wantInterval: time.Minute}, + {name: "cert only", yaml: &config.TLSYAMLConfig{CertFile: "y.crt"}, wantErr: true}, + {name: "key only", envKey: "e.key", wantErr: true}, + {name: "bad interval", envCert: "e.crt", envKey: "e.key", envInterval: "soon", wantErr: true}, + {name: "zero", envCert: "e.crt", envKey: "e.key", envInterval: "0s", wantErr: true}, + {name: "negative", envCert: "e.crt", envKey: "e.key", envInterval: "-1s", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := resolveTLS(tt.yaml, tt.envCert, tt.envKey, tt.envInterval, tt.flagCert, tt.flagKey) + if tt.wantErr { + if err == nil { + t.Fatal("wanted error") + } + return + } + if err != nil { + t.Fatal(err) + } + if tt.wantNil { + if got != nil { + t.Fatalf("got %#v", got) + } + return + } + if got == nil || got.CertFile != tt.wantCert || got.KeyFile != tt.wantKey || got.ReloadInterval != tt.wantInterval { + t.Fatalf("got %#v", got) + } + }) + } +} + +func TestServeHelpIncludesTLSFlags(t *testing.T) { + deps, _, stderr := testDeps() + if err := runServe([]string{"--help"}, deps); err != nil { + t.Fatal(err) + } + for _, name := range []string{"--tls-cert", "--tls-key"} { + if !strings.Contains(stderr.String(), name) { + t.Fatalf("missing %s", name) + } + } +} + +func TestWarnDisabledTLS(t *testing.T) { + tests := []struct { + name string + yaml *config.TLSYAMLConfig + resolved bool + wantWarn bool + }{ + {name: "absent section"}, + {name: "empty section", yaml: &config.TLSYAMLConfig{}, wantWarn: true}, + {name: "interval only", yaml: &config.TLSYAMLConfig{ReloadInterval: "60s"}, wantWarn: true}, + {name: "completed by env or flags", yaml: &config.TLSYAMLConfig{}, resolved: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var tlsCfg *server.TLSConfig + if tt.resolved { + tlsCfg = &server.TLSConfig{CertFile: "cert", KeyFile: "key", ReloadInterval: time.Minute} + } + var messages []string + warnDisabledTLS(tt.yaml, tlsCfg, func(format string, args ...any) { + messages = append(messages, fmt.Sprintf(format, args...)) + }) + if got := len(messages) == 1; got != tt.wantWarn { + t.Fatalf("warning emitted = %v, want %v; messages=%v", got, tt.wantWarn, messages) + } + if tt.wantWarn && !strings.Contains(messages[0], "serving plaintext HTTP") { + t.Fatalf("warning does not identify plaintext fallback: %q", messages[0]) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index a93dc13..ee6c4bb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -103,6 +103,13 @@ type ServerConfig struct { Callbacks *CallbacksConfig `yaml:"callbacks,omitempty"` Docs *bool `yaml:"docs,omitempty"` // enable /docs endpoint (default: true) ExternalURL string `yaml:"external_url,omitempty"` // public URL for OpenAPI docs (e.g. http://express-botx.invitro-dev.k8s) + TLS *TLSYAMLConfig `yaml:"tls,omitempty"` +} + +type TLSYAMLConfig struct { + CertFile string `yaml:"cert_file,omitempty"` + KeyFile string `yaml:"key_file,omitempty"` + ReloadInterval string `yaml:"reload_interval,omitempty"` } // CallbacksConfig holds settings for BotX callback handling. @@ -1088,6 +1095,10 @@ var knownKeys = map[string]map[string]bool{ "server": { "listen": true, "base_path": true, "api_keys": true, "allow_bot_secret_auth": true, "alertmanager": true, "grafana": true, "gitlab": true, "callbacks": true, "docs": true, "external_url": true, + "tls": true, + }, + "server.tls": { + "cert_file": true, "key_file": true, "reload_interval": true, }, "server.alertmanager": { "default_chat_id": true, "error_severities": true, "template": true, "template_file": true, diff --git a/internal/config/config_test.go b/internal/config/config_test.go index ea7600e..46f2319 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -521,6 +521,35 @@ func TestLoad_ServerConfig_Empty(t *testing.T) { } } +func TestLoad_ServerTLSConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte("bots:\n main:\n host: h\n id: b\n secret: s\nserver:\n tls:\n cert_file: /tls/tls.crt\n key_file: /tls/tls.key\n reload_interval: 45s\n") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(Flags{ConfigPath: path}) + if err != nil { + t.Fatal(err) + } + want := TLSYAMLConfig{CertFile: "/tls/tls.crt", KeyFile: "/tls/tls.key", ReloadInterval: "45s"} + if cfg.Server.TLS == nil || *cfg.Server.TLS != want { + t.Fatalf("TLS = %#v, want %#v", cfg.Server.TLS, want) + } +} + +func TestValidate_ServerTLSKeysAreKnown(t *testing.T) { + raw := []byte("server:\n tls:\n cert_file: /tls/tls.crt\n key_file: /tls/tls.key\n reload_interval: 60s\n") + var cfg Config + if err := yaml.Unmarshal(raw, &cfg); err != nil { + t.Fatal(err) + } + for _, result := range cfg.Validate(raw) { + if result.Level == ValidationWarning && strings.Contains(result.Message, "unknown key") { + t.Fatalf("TLS key reported unknown: %+v", result) + } + } +} + // --- ValidateFormat --- func TestValidateFormat(t *testing.T) { diff --git a/internal/server/server.go b/internal/server/server.go index 02fe531..3b872e7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -2,8 +2,10 @@ package server import ( "context" + "crypto/tls" "fmt" "log/slog" + "net" "net/http" "os" "strings" @@ -41,6 +43,14 @@ type Config struct { AsyncMode bool // when true, /send enqueues instead of sending directly DefaultRoutingMode string // default routing mode for async: direct, catalog, mixed MaxFileSize int64 // max file size in bytes for async mode (0 = default 1MB) + TLS *TLSConfig +} + +// TLSConfig configures HTTPS serving and certificate reloads. +type TLSConfig struct { + CertFile string + KeyFile string + ReloadInterval time.Duration } // Server is the HTTP server for express-botx. @@ -66,6 +76,12 @@ type Server struct { callbackCtx context.Context // cancelled on shutdown to signal async handlers callbackCancel context.CancelFunc // cancels callbackCtx srv *http.Server + tlsReloader *certReloader + ready chan struct{} + addrMu sync.RWMutex + addr net.Addr + pollerStarted func() + pollerDone chan struct{} } // SendFunc sends a message via the BotX API. The server calls this for each request. @@ -234,6 +250,10 @@ func New(cfg Config, sendFn SendFunc, chatResolver ChatResolver, opts ...Option) botNameSet: make(map[string]bool, len(cfg.BotNames)), callbackCtx: cbCtx, callbackCancel: cbCancel, + ready: make(chan struct{}), + } + if cfg.TLS != nil { + s.tlsReloader = newCertReloader(cfg.TLS.CertFile, cfg.TLS.KeyFile, cfg.TLS.ReloadInterval) } for _, k := range cfg.Keys { s.keyMap[k.Key] = k.Name @@ -353,6 +373,12 @@ func New(cfg Config, sendFn SendFunc, chatResolver ChatResolver, opts ...Option) Handler: handler, ReadHeaderTimeout: 10 * time.Second, } + if s.tlsReloader != nil { + s.srv.TLSConfig = &tls.Config{ + GetCertificate: s.tlsReloader.GetCertificate, + MinVersion: tls.VersionTLS12, + } + } s.srv.SetKeepAlivesEnabled(false) return s @@ -413,21 +439,87 @@ func (s *Server) resolveRequestBot(ctx context.Context, requestBot, chatBot stri return bot, "" } +// Ready returns a channel that closes after the listener is bound. +func (s *Server) Ready() <-chan struct{} { + return s.ready +} + +// Addr returns the bound listener address, or nil before a successful bind. +func (s *Server) Addr() net.Addr { + s.addrMu.RLock() + defer s.addrMu.RUnlock() + return s.addr +} + +func (s *Server) setAddr(addr net.Addr) { + s.addrMu.Lock() + s.addr = addr + s.addrMu.Unlock() +} + // Run starts the server and blocks until ctx is cancelled. It performs graceful shutdown. func (s *Server) Run(ctx context.Context) error { + runCtx, cancelRun := context.WithCancel(ctx) + defer cancelRun() defer s.callbackCancel() + if s.tlsReloader != nil { + if err := s.tlsReloader.loadInitial(); err != nil { + return fmt.Errorf("loading initial TLS certificate: %w", err) + } + } + + ln, err := net.Listen("tcp", s.cfg.Listen) + if err != nil { + return fmt.Errorf("listening on %s: %w", s.cfg.Listen, err) + } + defer func() { _ = ln.Close() }() + s.setAddr(ln.Addr()) + close(s.ready) + + if s.tlsReloader != nil { + s.pollerDone = make(chan struct{}) + if s.pollerStarted != nil { + s.pollerStarted() + } + go func() { + defer close(s.pollerDone) + s.tlsReloader.run(runCtx) + }() + defer func() { + cancelRun() + <-s.pollerDone + }() + } + + protocol := "http" + if s.tlsReloader != nil { + protocol = "https" + } + vlog.Info("server: listening on %s://%s (base_path: %s)", protocol, ln.Addr(), s.cfg.BasePath) + if len(s.keyMap) > 0 { + vlog.Info("server: %d API keys loaded", len(s.keyMap)) + } + errCh := make(chan error, 1) + serveDone := make(chan struct{}) go func() { - vlog.Info("server: listening on %s (base_path: %s)", s.cfg.Listen, s.cfg.BasePath) - if len(s.keyMap) > 0 { - vlog.Info("server: %d API keys loaded", len(s.keyMap)) + defer close(serveDone) + var serveErr error + if s.tlsReloader != nil { + serveErr = s.srv.ServeTLS(ln, "", "") + } else { + serveErr = s.srv.Serve(ln) } - if err := s.srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - errCh <- err + if serveErr != nil && serveErr != http.ErrServerClosed { + errCh <- serveErr } close(errCh) }() + defer func() { + _ = ln.Close() + <-serveDone + }() select { case err := <-errCh: @@ -435,21 +527,17 @@ func (s *Server) Run(ctx context.Context) error { case <-ctx.Done(): } - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelShutdown() vlog.Info("server: shutting down...") - // Stop accepting new connections first. var shutdownErr error if err := s.srv.Shutdown(shutdownCtx); err != nil { vlog.V1("server: HTTP shutdown error: %v", err) shutdownErr = err } - // Signal async callback handlers to stop. s.callbackCancel() - - // Wait for in-flight async callback handlers to finish (bounded by shutdownCtx). done := make(chan struct{}) go func() { s.callbackWG.Wait() @@ -464,7 +552,6 @@ func (s *Server) Run(ctx context.Context) error { shutdownErr = fmt.Errorf("server: timeout waiting for async callback handlers") } } - return shutdownErr } diff --git a/internal/server/server_tls_test.go b/internal/server/server_tls_test.go new file mode 100644 index 0000000..69bbd30 --- /dev/null +++ b/internal/server/server_tls_test.go @@ -0,0 +1,203 @@ +package server + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http" + "os" + "sync/atomic" + "testing" + "time" +) + +func startTestRun(t *testing.T, srv *Server) (context.CancelFunc, <-chan error) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + errs := make(chan error, 1) + go func() { errs <- srv.Run(ctx) }() + select { + case <-srv.Ready(): + case err := <-errs: + cancel() + t.Fatalf("Run before Ready: %v", err) + case <-time.After(2 * time.Second): + cancel() + t.Fatal("Ready timeout") + } + return cancel, errs +} + +func stopTestRun(t *testing.T, cancel context.CancelFunc, errs <-chan error) { + t.Helper() + cancel() + select { + case err := <-errs: + if err != nil { + t.Fatal(err) + } + case <-time.After(2 * time.Second): + t.Fatal("Run shutdown timeout") + } +} + +func testRoots(t *testing.T, certs ...[]byte) *x509.CertPool { + t.Helper() + pool := x509.NewCertPool() + for _, cert := range certs { + if !pool.AppendCertsFromPEM(cert) { + t.Fatal("append root") + } + } + return pool +} + +func tlsPeerSerial(addr string, roots *x509.CertPool, max uint16) (int64, error) { + conn, err := tls.Dial("tcp", addr, &tls.Config{ + RootCAs: roots, + ServerName: "127.0.0.1", + MinVersion: tls.VersionTLS10, + MaxVersion: max, + }) + if err != nil { + return 0, err + } + defer func() { _ = conn.Close() }() + return conn.ConnectionState().PeerCertificates[0].SerialNumber.Int64(), nil +} + +func TestServerRunHTTPRegression(t *testing.T) { + srv := newTestServer(nil, func(cfg *Config) { cfg.Listen = "127.0.0.1:0" }) + cancel, errs := startTestRun(t, srv) + resp, err := http.Get("http://" + srv.Addr().String() + "/healthz") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + stopTestRun(t, cancel, errs) +} + +func TestServerRunTLSAndMinimumVersion(t *testing.T) { + certFile, keyFile, certPEM, _, _ := writeTestPair(t, t.TempDir(), 1, nil) + srv := newTestServer(nil, func(cfg *Config) { + cfg.Listen = "127.0.0.1:0" + cfg.TLS = &TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: time.Minute} + }) + cancel, errs := startTestRun(t, srv) + roots := testRoots(t, certPEM) + client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{ + RootCAs: roots, + MinVersion: tls.VersionTLS12, + }}} + resp, err := client.Get("https://" + srv.Addr().String() + "/healthz") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d", resp.StatusCode) + } + if _, err := tlsPeerSerial(srv.Addr().String(), roots, tls.VersionTLS11); err == nil { + t.Fatal("TLS 1.1 succeeded") + } + stopTestRun(t, cancel, errs) +} + +func TestServerRunInitialFailureBeforeBind(t *testing.T) { + dir := t.TempDir() + certFile, keyFile := dir+"/tls.crt", dir+"/tls.key" + if err := os.WriteFile(certFile, []byte("bad"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, []byte("bad"), 0o600); err != nil { + t.Fatal(err) + } + srv := newTestServer(nil, func(cfg *Config) { + cfg.Listen = "127.0.0.1:0" + cfg.TLS = &TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: time.Minute} + }) + if err := srv.Run(context.Background()); err == nil { + t.Fatal("invalid keypair succeeded") + } + if srv.Addr() != nil { + t.Fatalf("listener bound at %v", srv.Addr()) + } +} + +func TestServerRunBindFailureDoesNotStartPoller(t *testing.T) { + occupied, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = occupied.Close() }() + certFile, keyFile, _, _, _ := writeTestPair(t, t.TempDir(), 1, nil) + srv := newTestServer(nil, func(cfg *Config) { + cfg.Listen = occupied.Addr().String() + cfg.TLS = &TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: time.Minute} + }) + var started atomic.Bool + srv.pollerStarted = func() { started.Store(true) } + if err := srv.Run(context.Background()); err == nil || started.Load() { + t.Fatalf("bind err=%v started=%v", err, started.Load()) + } +} + +func TestServerRunCancelsAndJoinsPoller(t *testing.T) { + certFile, keyFile, _, _, _ := writeTestPair(t, t.TempDir(), 1, nil) + srv := newTestServer(nil, func(cfg *Config) { + cfg.Listen = "127.0.0.1:0" + cfg.TLS = &TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: time.Minute} + }) + started := make(chan struct{}) + srv.pollerStarted = func() { close(started) } + cancel, errs := startTestRun(t, srv) + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("poller start timeout") + } + done := srv.pollerDone + if done == nil { + t.Fatal("pollerDone not stored") + } + stopTestRun(t, cancel, errs) + select { + case <-done: + default: + t.Fatal("Run returned before pollerDone closed") + } +} + +func TestServerRunHotReloadsCertificate(t *testing.T) { + dir := t.TempDir() + certFile, keyFile, cert1, _, _ := writeTestPair(t, dir, 1, nil) + cert2, key2, _ := generateTestPair(t, 2, nil) + srv := newTestServer(nil, func(cfg *Config) { + cfg.Listen = "127.0.0.1:0" + cfg.TLS = &TLSConfig{CertFile: certFile, KeyFile: keyFile, ReloadInterval: 10 * time.Millisecond} + }) + cancel, errs := startTestRun(t, srv) + if err := os.WriteFile(certFile, cert2, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, key2, 0o600); err != nil { + t.Fatal(err) + } + roots := testRoots(t, cert1, cert2) + deadline := time.Now().Add(2 * time.Second) + for { + serial, err := tlsPeerSerial(srv.Addr().String(), roots, tls.VersionTLS13) + if err == nil && serial == 2 { + break + } + if time.Now().After(deadline) { + t.Fatalf("reload timeout: serial=%d err=%v", serial, err) + } + time.Sleep(10 * time.Millisecond) + } + stopTestRun(t, cancel, errs) +} diff --git a/internal/server/tls.go b/internal/server/tls.go new file mode 100644 index 0000000..5024a3e --- /dev/null +++ b/internal/server/tls.go @@ -0,0 +1,122 @@ +package server + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "fmt" + "os" + "sync/atomic" + "time" + + vlog "github.com/lavr/express-botx/internal/log" +) + +const ( + defaultTLSReloadInterval = 60 * time.Second + stablePairReadAttempts = 3 +) + +type certReloader struct { + certFile string + keyFile string + reloadInterval time.Duration + cert atomic.Pointer[tls.Certificate] + lastCertHash [sha256.Size]byte + lastKeyHash [sha256.Size]byte + readFile func(string) ([]byte, error) +} + +func newCertReloader(certFile, keyFile string, interval time.Duration) *certReloader { + if interval <= 0 { + interval = defaultTLSReloadInterval + } + return &certReloader{ + certFile: certFile, + keyFile: keyFile, + reloadInterval: interval, + readFile: os.ReadFile, + } +} + +func (r *certReloader) readStablePair() ([]byte, []byte, error) { + for attempt := 1; attempt <= stablePairReadAttempts; attempt++ { + certBefore, err := r.readFile(r.certFile) + if err != nil { + return nil, nil, fmt.Errorf("reading TLS certificate %s: %w", r.certFile, err) + } + keyPEM, err := r.readFile(r.keyFile) + if err != nil { + return nil, nil, fmt.Errorf("reading TLS key %s: %w", r.keyFile, err) + } + certAfter, err := r.readFile(r.certFile) + if err != nil { + return nil, nil, fmt.Errorf("verifying TLS certificate %s: %w", r.certFile, err) + } + // Re-reading the certificate detects Kubernetes ..data swaps that + // straddle the key read. A key-only mixed/torn read that does not match + // the certificate is rejected later by tls.X509KeyPair, so it cannot + // replace the last-good pair. + if sha256.Sum256(certBefore) == sha256.Sum256(certAfter) { + return certAfter, keyPEM, nil + } + } + return nil, nil, fmt.Errorf("TLS certificate changed during %d stable-pair read attempts", stablePairReadAttempts) +} + +func (r *certReloader) reload() (bool, error) { + certPEM, keyPEM, err := r.readStablePair() + if err != nil { + return false, err + } + certHash, keyHash := sha256.Sum256(certPEM), sha256.Sum256(keyPEM) + if r.cert.Load() != nil && certHash == r.lastCertHash && keyHash == r.lastKeyHash { + return false, nil + } + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return false, fmt.Errorf("parsing TLS certificate/key pair: %w", err) + } + r.cert.Store(&cert) + r.lastCertHash, r.lastKeyHash = certHash, keyHash + return true, nil +} + +func (r *certReloader) loadInitial() error { + changed, err := r.reload() + if err != nil { + return err + } + if !changed || r.cert.Load() == nil { + return fmt.Errorf("initial TLS certificate was not loaded") + } + return nil +} + +func (r *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert := r.cert.Load() + if cert == nil { + return nil, fmt.Errorf("TLS certificate is not loaded") + } + return cert, nil +} + +func (r *certReloader) run(ctx context.Context) { + ticker := time.NewTicker(r.reloadInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + changed, err := r.reload() + if err != nil { + vlog.Info("server: TLS reload failed; keeping last good certificate: %v", err) + continue + } + if changed { + vlog.Info("server: TLS certificate reloaded") + } + } + } +} diff --git a/internal/server/tls_test.go b/internal/server/tls_test.go new file mode 100644 index 0000000..af5fb15 --- /dev/null +++ b/internal/server/tls_test.go @@ -0,0 +1,253 @@ +package server + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "runtime" + "sync" + "testing" + "time" +) + +func generateTestPair(t *testing.T, serial int64, key *rsa.PrivateKey) ([]byte, []byte, *rsa.PrivateKey) { + t.Helper() + if key == nil { + var err error + key, err = rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + } + template := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "127.0.0.1"}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), + pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), + key +} + +func writeTestPair(t *testing.T, dir string, serial int64, key *rsa.PrivateKey) (string, string, []byte, []byte, *rsa.PrivateKey) { + t.Helper() + certPEM, keyPEM, key := generateTestPair(t, serial, key) + certFile, keyFile := filepath.Join(dir, "tls.crt"), filepath.Join(dir, "tls.key") + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatal(err) + } + return certFile, keyFile, certPEM, keyPEM, key +} + +func servedSerial(t *testing.T, cert *tls.Certificate) int64 { + t.Helper() + parsed, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + return parsed.SerialNumber.Int64() +} + +func TestCertReloaderInitialAndUnchanged(t *testing.T) { + certFile, keyFile, _, _, _ := writeTestPair(t, t.TempDir(), 1, nil) + r := newCertReloader(certFile, keyFile, time.Minute) + if _, err := r.GetCertificate(nil); err == nil { + t.Fatal("GetCertificate before loadInitial succeeded") + } + if err := r.loadInitial(); err != nil { + t.Fatal(err) + } + before, _ := r.GetCertificate(nil) + changed, err := r.reload() + if err != nil || changed { + t.Fatalf("reload unchanged = (%v, %v)", changed, err) + } + after, _ := r.GetCertificate(nil) + if after != before { + t.Fatal("unchanged content replaced pointer") + } +} + +func TestCertReloaderReloadAndLastGood(t *testing.T) { + dir := t.TempDir() + certFile, keyFile, _, _, _ := writeTestPair(t, dir, 1, nil) + r := newCertReloader(certFile, keyFile, time.Minute) + if err := r.loadInitial(); err != nil { + t.Fatal(err) + } + _, _, _, _, _ = writeTestPair(t, dir, 2, nil) + changed, err := r.reload() + if err != nil || !changed { + t.Fatalf("reload = (%v, %v)", changed, err) + } + current, _ := r.GetCertificate(nil) + if got := servedSerial(t, current); got != 2 { + t.Fatalf("serial = %d, want 2", got) + } + before := current + certHash, keyHash := r.lastCertHash, r.lastKeyHash + if err := os.WriteFile(keyFile, []byte("broken key"), 0o600); err != nil { + t.Fatal(err) + } + changed, err = r.reload() + if err == nil || changed { + t.Fatalf("broken reload = (%v, %v)", changed, err) + } + after, _ := r.GetCertificate(nil) + if after != before || r.lastCertHash != certHash || r.lastKeyHash != keyHash { + t.Fatal("failed reload changed last-good state") + } +} + +func TestCertReloaderHashesFilesSeparately(t *testing.T) { + dir := t.TempDir() + certPEM, keyPEM, key := generateTestPair(t, 1, nil) + certFile, keyFile := filepath.Join(dir, "tls.crt"), filepath.Join(dir, "tls.key") + extra := pem.EncodeToMemory(&pem.Block{Type: "IGNORED", Bytes: []byte("block")}) + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, append(append([]byte{}, extra...), keyPEM...), 0o600); err != nil { + t.Fatal(err) + } + r := newCertReloader(certFile, keyFile, time.Minute) + if err := r.loadInitial(); err != nil { + t.Fatal(err) + } + + renewedCert, _, _ := generateTestPair(t, 2, key) + if err := os.WriteFile(certFile, renewedCert, 0o600); err != nil { + t.Fatal(err) + } + if changed, err := r.reload(); err != nil || !changed { + t.Fatalf("cert-only change = (%v, %v)", changed, err) + } + + // Restore the exact initial concatenation and make it the stored baseline. + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, append(append([]byte{}, extra...), keyPEM...), 0o600); err != nil { + t.Fatal(err) + } + if changed, err := r.reload(); err != nil || !changed { + t.Fatalf("restore concatenation baseline = (%v, %v)", changed, err) + } + + // certPEM || extra || keyPEM is unchanged while each file changes. + if err := os.WriteFile(certFile, append(append([]byte{}, certPEM...), extra...), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatal(err) + } + if changed, err := r.reload(); err != nil || !changed { + t.Fatalf("concatenation ambiguity = (%v, %v)", changed, err) + } +} + +func kubeSecretMount(t *testing.T) (string, string, func()) { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("Kubernetes projection test requires symlinks") + } + mount := t.TempDir() + for _, item := range []struct { + dir string + serial int64 + }{{"..data_v1", 1}, {"..data_v2", 2}} { + path := filepath.Join(mount, item.dir) + if err := os.Mkdir(path, 0o700); err != nil { + t.Fatal(err) + } + cert, key, _ := generateTestPair(t, item.serial, nil) + if err := os.WriteFile(filepath.Join(path, "tls.crt"), cert, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "tls.key"), key, 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink("..data_v1", filepath.Join(mount, "..data")); err != nil { + t.Skipf("symlink unavailable: %v", err) + } + if err := os.Symlink(filepath.Join("..data", "tls.crt"), filepath.Join(mount, "tls.crt")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join("..data", "tls.key"), filepath.Join(mount, "tls.key")); err != nil { + t.Fatal(err) + } + swap := func() { + tmp := filepath.Join(mount, "..data_tmp") + if err := os.Symlink("..data_v2", tmp); err != nil { + t.Fatal(err) + } + if err := os.Rename(tmp, filepath.Join(mount, "..data")); err != nil { + t.Fatal(err) + } + } + return filepath.Join(mount, "tls.crt"), filepath.Join(mount, "tls.key"), swap +} + +func TestCertReloaderKubernetesSwapAndMidReadRetry(t *testing.T) { + for _, midRead := range []bool{false, true} { + t.Run(map[bool]string{false: "between reloads", true: "between cert and key reads"}[midRead], func(t *testing.T) { + certFile, keyFile, swap := kubeSecretMount(t) + r := newCertReloader(certFile, keyFile, time.Minute) + if err := r.loadInitial(); err != nil { + t.Fatal(err) + } + if midRead { + var once sync.Once + r.readFile = func(name string) ([]byte, error) { + data, err := os.ReadFile(name) + if err == nil && name == certFile { + once.Do(swap) + } + return data, err + } + } else { + swap() + } + changed, err := r.reload() + if err != nil || !changed { + t.Fatalf("reload = (%v, %v)", changed, err) + } + cert, _ := r.GetCertificate(nil) + if got := servedSerial(t, cert); got != 2 { + t.Fatalf("mixed/stale serial = %d, want 2", got) + } + }) + } +} + +func TestCertReloaderClampsInterval(t *testing.T) { + for _, interval := range []time.Duration{0, -time.Second} { + r := newCertReloader("cert", "key", interval) + if r.reloadInterval != defaultTLSReloadInterval { + t.Fatalf("interval = %v", r.reloadInterval) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r.run(ctx) + } +}